diff --git a/backend/agenteval/evaluation/cost_tracking.py b/backend/agenteval/evaluation/cost_tracking.py index ea41a44..5297581 100644 --- a/backend/agenteval/evaluation/cost_tracking.py +++ b/backend/agenteval/evaluation/cost_tracking.py @@ -1,9 +1,21 @@ -"""Cost tracking and calculation for evaluation runs.""" +"""Cost tracking for evaluation runs — pricing lookup and usage-based cost. -from typing import Any +成本口径:评测自身消耗的 LLM token(judge/generator/embedding/moderation, +经 ModelGateway 归集进 RunSummary.eval_usage_by_purpose),按岗位所用模型的 +单价计费。被评智能体的通道用量不在口径内(tutu-api 通道不返回用量)。 + +单价解析顺序:``data/model_pricing.json``(部署侧覆盖)→ ``DEFAULT_PRICING``。 +""" + +import json +from pathlib import Path +from typing import Any, Optional from pydantic import BaseModel, Field +# data/model_pricing.json:与 DB 同目录,部署时挂载覆盖,缺省文件视为无覆盖 +_PRICING_FILE = Path(__file__).resolve().parents[3] / "data" / "model_pricing.json" + class ModelPricing(BaseModel): """Pricing for a model (per 1M tokens).""" @@ -13,16 +25,8 @@ class ModelPricing(BaseModel): completion_cost_per_1m: float = Field(description="Cost per 1M completion tokens in USD") -class TokenUsage(BaseModel): - """Token usage for a single API call.""" - - prompt_tokens: int = 0 - completion_tokens: int = 0 - total_tokens: int = 0 - - class CostBreakdown(BaseModel): - """Cost breakdown for a turn, case, or run.""" + """Cost breakdown for one usage bucket (e.g. one purpose or a whole run).""" prompt_tokens: int = 0 completion_tokens: int = 0 @@ -40,118 +44,99 @@ DEFAULT_PRICING: dict[str, ModelPricing] = { "claude-3-haiku": ModelPricing(model_id="claude-3-haiku", prompt_cost_per_1m=0.25, completion_cost_per_1m=1.25), } +_pricing_overrides: Optional[dict[str, ModelPricing]] = None + + +def _load_overrides() -> dict[str, ModelPricing]: + """Parse data/model_pricing.json once; a missing/broken file means no override.""" + global _pricing_overrides + if _pricing_overrides is not None: + return _pricing_overrides + overrides: dict[str, ModelPricing] = {} + try: + raw = json.loads(_PRICING_FILE.read_text(encoding="utf-8")) + if isinstance(raw, dict): + for name, entry in raw.items(): + try: + overrides[str(name)] = ModelPricing(model_id=str(name), **(entry or {})) + except Exception: + continue + except (OSError, ValueError): + pass + _pricing_overrides = overrides + return overrides + + +def reload_pricing_overrides() -> None: + """Drop the cached overrides so the next get_pricing() re-reads the file.""" + global _pricing_overrides + _pricing_overrides = None + + +def get_pricing(model_name: Optional[str]) -> Optional[ModelPricing]: + """Resolve pricing for a model name; None when the model is unknown.""" + if not model_name: + return None + return _load_overrides().get(model_name) or DEFAULT_PRICING.get(model_name) + def calculate_cost( prompt_tokens: int, completion_tokens: int, pricing: ModelPricing, ) -> float: - """Calculate cost in USD for given token usage and pricing. - - Args: - prompt_tokens: Number of prompt tokens - completion_tokens: Number of completion tokens - pricing: Model pricing configuration - - Returns: - Cost in USD - """ + """Calculate cost in USD for given token usage and pricing.""" prompt_cost = (prompt_tokens / 1_000_000) * pricing.prompt_cost_per_1m completion_cost = (completion_tokens / 1_000_000) * pricing.completion_cost_per_1m return prompt_cost + completion_cost -def aggregate_token_usage(turns: list[dict[str, Any]]) -> TokenUsage: - """Aggregate token usage from a list of turns. - - Args: - turns: List of turn dicts with optional token fields - - Returns: - Aggregated token usage - """ - prompt_tokens = sum(t.get("prompt_tokens") or 0 for t in turns) - completion_tokens = sum(t.get("completion_tokens") or 0 for t in turns) - return TokenUsage( +def usage_breakdown(usage: dict[str, Any], pricing: ModelPricing) -> CostBreakdown: + """Turn one recorded usage dict (prompt/completion tokens) into a CostBreakdown.""" + prompt_tokens = int(usage.get("prompt_tokens") or 0) + completion_tokens = int(usage.get("completion_tokens") or 0) + return CostBreakdown( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, + cost_usd=calculate_cost(prompt_tokens, completion_tokens, pricing), ) -def calculate_turn_cost( - turn: dict[str, Any], - pricing: ModelPricing, -) -> CostBreakdown: - """Calculate cost for a single turn. +def build_eval_cost_section( + usage_by_purpose: Optional[dict[str, dict[str, Any]]], + model_configs: Optional[dict[str, Any]], +) -> Optional[dict[str, Any]]: + """Per-purpose eval-LLM cost from recorded run usage; None when no usage. - Args: - turn: Turn dict with optional token fields - pricing: Model pricing configuration - - Returns: - Cost breakdown for the turn + ``model_configs`` is the run summary's purpose → model snapshot map; a purpose + whose model has no known pricing gets ``cost_usd: None`` (tokens still shown). """ - prompt_tokens = turn.get("prompt_tokens") or 0 - completion_tokens = turn.get("completion_tokens") or 0 - total_tokens = prompt_tokens + completion_tokens - cost_usd = calculate_cost(prompt_tokens, completion_tokens, pricing) - - return CostBreakdown( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - cost_usd=cost_usd, - ) - - -def calculate_case_cost( - turns: list[dict[str, Any]], - pricing: ModelPricing, -) -> CostBreakdown: - """Calculate cost for a case (multiple turns). - - Args: - turns: List of turn dicts - pricing: Model pricing configuration - - Returns: - Aggregated cost breakdown for the case - """ - usage = aggregate_token_usage(turns) - cost_usd = calculate_cost(usage.prompt_tokens, usage.completion_tokens, pricing) - - return CostBreakdown( - prompt_tokens=usage.prompt_tokens, - completion_tokens=usage.completion_tokens, - total_tokens=usage.total_tokens, - cost_usd=cost_usd, - ) - - -def calculate_run_cost( - cases: list[dict[str, Any]], - pricing: ModelPricing, -) -> CostBreakdown: - """Calculate total cost for a run (multiple cases). - - Args: - cases: List of case dicts, each with a 'turns' field - pricing: Model pricing configuration - - Returns: - Aggregated cost breakdown for the run - """ - all_turns = [] - for case in cases: - all_turns.extend(case.get("turns", [])) - - usage = aggregate_token_usage(all_turns) - cost_usd = calculate_cost(usage.prompt_tokens, usage.completion_tokens, pricing) - - return CostBreakdown( - prompt_tokens=usage.prompt_tokens, - completion_tokens=usage.completion_tokens, - total_tokens=usage.total_tokens, - cost_usd=cost_usd, - ) + if not usage_by_purpose: + return None + model_configs = model_configs or {} + items: list[dict[str, Any]] = [] + total_cost = 0.0 + has_any_cost = False + for purpose, usage in sorted(usage_by_purpose.items()): + model_name = (model_configs.get(purpose) or {}).get("model_name") + pricing = get_pricing(model_name) + breakdown = usage_breakdown(usage, pricing) if pricing else None + if breakdown is not None: + total_cost += breakdown.cost_usd + has_any_cost = True + items.append( + { + "purpose": purpose, + "model_name": model_name, + "prompt_tokens": breakdown.prompt_tokens if breakdown else int(usage.get("prompt_tokens") or 0), + "completion_tokens": breakdown.completion_tokens if breakdown else int(usage.get("completion_tokens") or 0), + "total_tokens": breakdown.total_tokens if breakdown else int(usage.get("total_tokens") or 0), + "cost_usd": breakdown.cost_usd if breakdown else None, + } + ) + return { + "by_purpose": items, + "total_tokens": sum(i["total_tokens"] for i in items), + "total_cost_usd": round(total_cost, 6) if has_any_cost else None, + } diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index 373af8f..d03acf3 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -118,6 +118,9 @@ class EvalEngine: max(1, get_settings().max_concurrent_rules), ) self._state_lock = asyncio.Lock() + # 分岗位的评测侧 token 用量(ModelPurpose.value → 用量),由各规则/生成器 + # 的按次用量归集而来,避免并发下用网关总量做差产生竞态。 + self._usage_by_purpose: dict[str, dict[str, int]] = {} # Collects fatal case-level errors (e.g. dynamic message generation # failures) so their cause is persisted into run.summary — not just # emitted transiently over WebSocket. @@ -216,6 +219,7 @@ class EvalEngine: case_errors=self._case_errors or None, model_configs=resolved_snapshot or None, eval_token_usage=usage if usage["total_tokens"] > 0 else None, + eval_usage_by_purpose=self._usage_by_purpose or None, ) run.status = RunStatus.COMPLETED run.completed_at = utc_now() @@ -479,6 +483,8 @@ class EvalEngine: gateway=self.model_gateway if model_config else None, ) result = await rule.evaluate(case, dialog) + if purpose: + await self._accumulate_usage(purpose.value, rule.llm_usage) except Exception as exc: result = RuleResult(passed=False, reason=f"模型配置解析失败: {exc}") return rule_config, is_implicit, result @@ -571,7 +577,10 @@ class EvalEngine: try: model_config = await self._resolve_model(ModelPurpose.GENERATOR) if model_config: - content = await self.model_gateway.chat(model_config, messages_payload, temperature=0.7) + content, usage = await self.model_gateway.chat_with_usage( + model_config, messages_payload, temperature=0.7 + ) + await self._accumulate_usage(ModelPurpose.GENERATOR.value, usage) else: content = await self._generate_messages_legacy(messages_payload) @@ -625,6 +634,16 @@ class EvalEngine: raise ValueError("LLM 返回内容为空或无法解析") return content + async def _accumulate_usage(self, purpose_key: str, usage: dict[str, int] | None) -> None: + if not usage or not usage.get("total_tokens"): + return + async with self._state_lock: + bucket = self._usage_by_purpose.setdefault( + purpose_key, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + ) + for key in bucket: + bucket[key] += int(usage.get(key) or 0) + async def _resolve_model(self, purpose: ModelPurpose | None) -> ModelRuntimeConfig | None: if purpose is None: return None diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index 1786490..5016eae 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -12,6 +12,7 @@ from typing import Any, Optional from sqlmodel import Session from agenteval.evaluation.case_verdict import build_case_evidence, resolve_case_verdicts +from agenteval.evaluation.cost_tracking import build_eval_cost_section from agenteval.evaluation.go_no_go import AcceptanceCriteria, evaluate_go_no_go from agenteval.evaluation.metrics import aggregate_runs from agenteval.evaluation.report_render import render_html, render_json, render_markdown @@ -117,6 +118,8 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: "judged_pass_rate": judged_pass_rate, "avg_latency_ms": summary.avg_latency_ms, "eval_token_usage": summary.eval_token_usage, + "eval_usage_by_purpose": summary.eval_usage_by_purpose, + "eval_cost": build_eval_cost_section(summary.eval_usage_by_purpose, summary.model_configs), } # Generate go/no-go verdict(场景级验收标准优先,缺省用全局默认) diff --git a/backend/agenteval/evaluation/report_render.py b/backend/agenteval/evaluation/report_render.py index 5c5616b..7120d4b 100644 --- a/backend/agenteval/evaluation/report_render.py +++ b/backend/agenteval/evaluation/report_render.py @@ -33,6 +33,12 @@ HTML_TEMPLATE = """ .badge { padding: 2px 8px; border-radius: 4px; font-size: 12px; } .pass { background: #e6f7e6; color: #2e7d32; } .fail { background: #ffebee; color: #c62828; } + .verdict { border-radius: 6px; padding: 16px; margin: 24px 0; } + .verdict .decision { font-size: 18px; font-weight: 700; } + .verdict ul { margin: 8px 0 0; padding-left: 20px; } + .verdict-go { background: #e6f7e6; border: 1px solid #b7e0b8; color: #2e7d32; } + .verdict-no_go { background: #ffebee; border: 1px solid #f5c6cb; color: #c62828; } + .verdict-conditional { background: #fff8e1; border: 1px solid #ffe082; color: #8d6e00; } pre { white-space: pre-wrap; word-break: break-word; background: #f5f5f5; padding: 8px; border-radius: 4px; } @@ -43,6 +49,22 @@ HTML_TEMPLATE = """

评测场景:{{ report.scenario_name }}({{ report.scenario_id }})

执行时间:{{ report.started_at }} 至 {{ report.completed_at or '进行中' }}

+ {% if report.go_no_go %} + {% set gng = report.go_no_go %} + {% set decision_label = {'go': 'GO — 建议上线', 'no_go': 'NO-GO — 不建议上线', 'conditional': '有条件通过 — 修复后复测'} %} +
+
上线评估:{{ decision_label.get(gng.decision, gng.decision) }}
+
{{ gng.summary }}
+ {% if gng.criteria_results %} + + {% endif %} +
+ {% endif %} +
{{ report.summary.total_cases }}
@@ -121,6 +143,25 @@ def render_markdown(report: dict[str, Any]) -> str: f"**开始时间**: {report.get('started_at', '-')} ", f"**完成时间**: {report.get('completed_at', '-')} ", "", + ] + + gng = report.get("go_no_go") + if gng: + decision_label = { + "go": "GO — 建议上线", + "no_go": "NO-GO — 不建议上线", + "conditional": "有条件通过 — 修复后复测", + } + lines += [ + f"> **上线评估:{decision_label.get(gng.get('decision'), gng.get('decision'))}**", + f"> {gng.get('summary', '')}", + ] + for c in gng.get("criteria_results") or []: + mark = "✅" if c.get("passed") else "❌" + lines.append(f"> {mark} {c.get('detail', '')}") + lines.append("") + + lines += [ "## 汇总", "", "| 指标 | 数值 |", @@ -134,6 +175,26 @@ def render_markdown(report: dict[str, Any]) -> str: f"| 连通用例 | {s.get('connectivity_cases', 0)} |", f"| 判定型通过率 | {judged_rate_text} |", "", + ] + + cost = s.get("eval_cost") + if cost: + purpose_labels = {"judge": "评分判定", "generator": "用例生成", "embedding": "语义向量", "moderation": "安全审核"} + lines += [ + "## 评测成本", + "", + "| 岗位 | 模型 | Token | 成本 (USD) |", + "|------|------|------|------|", + ] + for item in cost.get("by_purpose") or []: + label = purpose_labels.get(item.get("purpose"), item.get("purpose") or "-") + cost_text = "—" if item.get("cost_usd") is None else f"${item['cost_usd']:.6f}" + lines.append(f"| {label} | {item.get('model_name') or '-'} | {item.get('total_tokens', 0)} | {cost_text} |") + total_cost = cost.get("total_cost_usd") + total_text = "—" if total_cost is None else f"${total_cost:.6f}" + lines += [f"| **合计** | | {cost.get('total_tokens', 0)} | {total_text} |", ""] + + lines += [ "## 用例明细", "", ] diff --git a/backend/agenteval/evaluation/rules/base.py b/backend/agenteval/evaluation/rules/base.py index cb8059c..c8b60ff 100644 --- a/backend/agenteval/evaluation/rules/base.py +++ b/backend/agenteval/evaluation/rules/base.py @@ -35,6 +35,14 @@ class EvalRule(ABC): self.params = params self.model_config = model_config self.gateway = gateway + # 本规则执行期间消耗的评测侧 token 用量(引擎按评测岗位归集) + self.llm_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + def _record_llm_usage(self, usage: Optional[dict[str, int]]) -> None: + if not usage: + return + for key in self.llm_usage: + self.llm_usage[key] += int(usage.get(key) or 0) @abstractmethod async def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult: diff --git a/backend/agenteval/evaluation/rules/fluency.py b/backend/agenteval/evaluation/rules/fluency.py index baff357..f005cf6 100644 --- a/backend/agenteval/evaluation/rules/fluency.py +++ b/backend/agenteval/evaluation/rules/fluency.py @@ -3,6 +3,7 @@ import json from agenteval.evaluation.rules.base import EvalRule, RuleResult, register_rule +from agenteval.evaluation.rules.scored_llm import call_scored_llm, parse_scored_content from agenteval.models import Case, Turn from agenteval.utils.llm import extract_reply_text @@ -88,7 +89,7 @@ class FluencyRule(EvalRule): """Evaluate conversation fluency using the gateway.""" system_prompt, user_prompt = self._build_prompts(conversation, custom_criteria) try: - content = await self.gateway.chat( + content, usage = await self.gateway.chat_with_usage( self.model_config, [ {"role": "system", "content": system_prompt}, @@ -96,7 +97,8 @@ class FluencyRule(EvalRule): ], temperature=0.2, ) - return self._parse_score(content) + self._record_llm_usage(usage) + return parse_scored_content(content) except Exception as exc: return None, str(exc) @@ -109,35 +111,8 @@ class FluencyRule(EvalRule): custom_criteria: str, ) -> tuple[float | None, str]: """Call LLM API directly for fluency assessment.""" - import httpx - system_prompt, user_prompt = self._build_prompts(conversation, custom_criteria) - - headers = {"Content-Type": "application/json"} - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - - payload = { - "model": model, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}, - ], - "temperature": 0.2, - } - - try: - async with httpx.AsyncClient(timeout=60) as client: - resp = await client.post(api_url, headers=headers, json=payload) - resp.raise_for_status() - from agenteval.utils.llm import extract_content_from_llm_response - - content = extract_content_from_llm_response(resp.json()) - if not content: - return None, "LLM 返回内容为空" - return self._parse_score(content) - except Exception as exc: - return None, str(exc) + return await call_scored_llm(api_url, api_key, model, system_prompt, user_prompt) @staticmethod def _build_prompts(conversation: str, custom_criteria: str) -> tuple[str, str]: @@ -158,12 +133,3 @@ class FluencyRule(EvalRule): ) user_prompt = f"对话内容:\n{conversation}" return system_prompt, user_prompt - - @staticmethod - def _parse_score(content: str) -> tuple[float, str]: - """Parse LLM response to extract score and reason.""" - from agenteval.utils.llm import parse_json_from_llm_text - - parsed = parse_json_from_llm_text(content) - score = float(parsed["score"]) - return max(0.0, min(10.0, score)), parsed.get("reason", "") diff --git a/backend/agenteval/evaluation/rules/llm_score.py b/backend/agenteval/evaluation/rules/llm_score.py index 9ec0ae7..ececc77 100644 --- a/backend/agenteval/evaluation/rules/llm_score.py +++ b/backend/agenteval/evaluation/rules/llm_score.py @@ -3,11 +3,10 @@ import asyncio import json -import httpx - from agenteval.evaluation.rules.base import EvalRule, RuleResult, register_rule +from agenteval.evaluation.rules.scored_llm import call_scored_llm, parse_scored_content from agenteval.models import Case, Turn -from agenteval.utils.llm import extract_content_from_llm_response, extract_reply_text, parse_json_from_llm_text +from agenteval.utils.llm import extract_reply_text @register_rule @@ -120,7 +119,7 @@ class LlmScoreRule(EvalRule): async def _call_gateway(self, question: str, reply: str, criteria: str) -> tuple[float | None, str]: system_prompt, user_prompt = self._prompts(question, reply, criteria) try: - content = await self.gateway.chat( + content, usage = await self.gateway.chat_with_usage( self.model_config, [ {"role": "system", "content": system_prompt}, @@ -128,7 +127,8 @@ class LlmScoreRule(EvalRule): ], temperature=0.2, ) - return self._parse_score(content) + self._record_llm_usage(usage) + return parse_scored_content(content) except Exception as exc: return None, str(exc) @@ -141,12 +141,6 @@ class LlmScoreRule(EvalRule): ) return system_prompt, f"用户问题:{question}\n智能体回复:{reply}" - @staticmethod - def _parse_score(content: str) -> tuple[float, str]: - parsed = parse_json_from_llm_text(content) - score = float(parsed["score"]) - return max(0.0, min(10.0, score)), parsed.get("reason", "") - async def _call_llm( self, api_url: str, @@ -158,28 +152,4 @@ class LlmScoreRule(EvalRule): ) -> tuple[float | None, str]: """Call the configured LLM API and parse a numeric score between 0 and 10.""" system_prompt, user_prompt = self._prompts(question, reply, criteria) - - headers = {"Content-Type": "application/json"} - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - - payload = { - "model": model, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}, - ], - "temperature": 0.2, - } - - try: - async with httpx.AsyncClient(timeout=60) as client: - resp = await client.post(api_url, headers=headers, json=payload) - resp.raise_for_status() - content = extract_content_from_llm_response(resp.json()) - if not content: - return None, "LLM 返回内容为空" - - return self._parse_score(content) - except Exception as exc: - return None, str(exc) + return await call_scored_llm(api_url, api_key, model, system_prompt, user_prompt) diff --git a/backend/agenteval/evaluation/rules/safety.py b/backend/agenteval/evaluation/rules/safety.py index 0f14a8b..07a36e3 100644 --- a/backend/agenteval/evaluation/rules/safety.py +++ b/backend/agenteval/evaluation/rules/safety.py @@ -146,7 +146,7 @@ class SafetyRule(EvalRule): user_prompt = f"对话上下文:\n{context}\n\n助手最后回复:{reply_text}" try: - content = await self.gateway.chat( + content, usage = await self.gateway.chat_with_usage( self.model_config, [ {"role": "system", "content": system_prompt}, @@ -154,6 +154,7 @@ class SafetyRule(EvalRule): ], temperature=0.1, ) + self._record_llm_usage(usage) parsed = parse_json_from_llm_text(content) if parsed.get("hallucination", False): return parsed.get("reason", "疑似编造事实") diff --git a/backend/agenteval/evaluation/rules/scored_llm.py b/backend/agenteval/evaluation/rules/scored_llm.py new file mode 100644 index 0000000..c50dc6b --- /dev/null +++ b/backend/agenteval/evaluation/rules/scored_llm.py @@ -0,0 +1,51 @@ +"""Shared plumbing for rules that ask an LLM for a 0-10 score. + +llm_score 与 fluency 共用的直连 LLM 调用与评分解析——两处各自实现一遍的 +重复逻辑收敛到这里(v1.3.1 Phase 3 代码质量项)。 +""" + +import httpx + +from agenteval.utils.llm import extract_content_from_llm_response, parse_json_from_llm_text + + +def parse_scored_content(content: str) -> tuple[float, str]: + """Parse an LLM reply into a clamped 0-10 score plus reason.""" + parsed = parse_json_from_llm_text(content) + score = float(parsed["score"]) + return max(0.0, min(10.0, score)), parsed.get("reason", "") + + +async def call_scored_llm( + api_url: str, + api_key: str | None, + model: str, + system_prompt: str, + user_prompt: str, + timeout: float = 60.0, +) -> tuple[float | None, str]: + """POST a chat request to a raw OpenAI-compatible endpoint and parse the score. + + Returns (score, reason); score is None with an error message on failure. + """ + headers = {"Content-Type": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + payload = { + "model": model, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + "temperature": 0.2, + } + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post(api_url, headers=headers, json=payload) + resp.raise_for_status() + content = extract_content_from_llm_response(resp.json()) + if not content: + return None, "LLM 返回内容为空" + return parse_scored_content(content) + except Exception as exc: + return None, str(exc) diff --git a/backend/agenteval/evaluation/rules/semantic.py b/backend/agenteval/evaluation/rules/semantic.py index 48aca69..31b359f 100644 --- a/backend/agenteval/evaluation/rules/semantic.py +++ b/backend/agenteval/evaluation/rules/semantic.py @@ -74,7 +74,10 @@ class SemanticSimilarityRule(EvalRule): try: if self.model_config and self.gateway: - reply_vec, ref_vec = await self.gateway.embed(self.model_config, [reply_text, reference]) + (reply_vec, ref_vec), usage = await self.gateway.embed_with_usage( + self.model_config, [reply_text, reference] + ) + self._record_llm_usage(usage) else: api_url: str | None = self.params.get("api_url") api_key: str | None = self.params.get("api_key") diff --git a/backend/agenteval/evaluation/run_summary.py b/backend/agenteval/evaluation/run_summary.py index 3d92b30..e45356b 100644 --- a/backend/agenteval/evaluation/run_summary.py +++ b/backend/agenteval/evaluation/run_summary.py @@ -21,6 +21,7 @@ def build_run_summary( case_errors: Optional[list[dict[str, str]]] = None, model_configs: Optional[dict[str, Any]] = None, eval_token_usage: Optional[dict[str, int]] = None, + eval_usage_by_purpose: Optional[dict[str, dict[str, int]]] = None, ) -> RunSummary: """Compute a run's summary口径 from its authoritative case outcomes.""" total_cases = len(case_outcomes) @@ -50,6 +51,7 @@ def build_run_summary( abandonment_rate=abandonment_rate, avg_latency_ms=avg_latency_ms, eval_token_usage=eval_token_usage, + eval_usage_by_purpose=eval_usage_by_purpose, case_outcomes={ case_id: CaseOutcomeSummary( passed=o.passed, connectivity=o.connectivity, abandoned=o.abandoned diff --git a/backend/agenteval/model_gateway.py b/backend/agenteval/model_gateway.py index 8bdb5f7..921bafb 100644 --- a/backend/agenteval/model_gateway.py +++ b/backend/agenteval/model_gateway.py @@ -4,7 +4,7 @@ from typing import Any import httpx -from agenteval.model_protocols import ModelProtocolAdapter, ProtocolAdapterError, get_protocol_adapter +from agenteval.model_protocols import ProtocolAdapterError, get_protocol_adapter from agenteval.services.model_configs import ModelRuntimeConfig @@ -19,13 +19,6 @@ class ModelGateway: # 评测侧 LLM 调用的累计 token 用量(引擎结束时写入 run summary) self.total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} - def _record_usage(self, adapter: ModelProtocolAdapter, data: dict[str, Any]) -> None: - usage = adapter.parse_usage(data) - if not usage: - return - for key in self.total_usage: - self.total_usage[key] += int(usage.get(key) or 0) - async def _post(self, config: ModelRuntimeConfig, payload: dict[str, Any]) -> dict[str, Any]: adapter = self._adapter(config) try: @@ -61,25 +54,49 @@ class ModelGateway: messages: list[dict[str, str]], temperature: float = 0.2, ) -> str: + content, _ = await self.chat_with_usage(config, messages, temperature) + return content + + async def chat_with_usage( + self, + config: ModelRuntimeConfig, + messages: list[dict[str, str]], + temperature: float = 0.2, + ) -> tuple[str, dict[str, int] | None]: + """Like chat(), but also returns this call's token usage (None if omitted).""" adapter = self._adapter(config) try: payload = adapter.chat_payload(config.model_name, messages, temperature) data = await self._post(config, payload) - self._record_usage(adapter, data) - return adapter.parse_chat(data) + usage = adapter.parse_usage(data) + self._accumulate(usage) + return adapter.parse_chat(data), usage except ProtocolAdapterError as exc: raise ModelGatewayError(str(exc)) from exc async def embed(self, config: ModelRuntimeConfig, inputs: str | list[str]) -> list[list[float]]: + vectors, _ = await self.embed_with_usage(config, inputs) + return vectors + + async def embed_with_usage( + self, config: ModelRuntimeConfig, inputs: str | list[str] + ) -> tuple[list[list[float]], dict[str, int] | None]: adapter = self._adapter(config) try: payload = adapter.embedding_payload(config.model_name, inputs) data = await self._post(config, payload) - self._record_usage(adapter, data) - return adapter.parse_embeddings(data) + usage = adapter.parse_usage(data) + self._accumulate(usage) + return adapter.parse_embeddings(data), usage except ProtocolAdapterError as exc: raise ModelGatewayError(str(exc)) from exc + def _accumulate(self, usage: dict[str, int] | None) -> None: + if not usage: + return + for key in self.total_usage: + self.total_usage[key] += int(usage.get(key) or 0) + async def moderate(self, config: ModelRuntimeConfig, text: str) -> dict[str, Any]: adapter = self._adapter(config) try: diff --git a/backend/agenteval/models.py b/backend/agenteval/models.py index af48922..8ef11eb 100644 --- a/backend/agenteval/models.py +++ b/backend/agenteval/models.py @@ -200,6 +200,8 @@ class RunSummary(BaseModel): avg_latency_ms: Optional[float] = None # 评测侧 LLM 调用累计 token 用量(judge/generator 等,引擎经 ModelGateway 统计) eval_token_usage: Optional[dict[str, int]] = None + # 分岗位用量(ModelPurpose.value → 用量),成本核算按岗位 × 模型计价 + eval_usage_by_purpose: Optional[dict[str, dict[str, int]]] = None case_outcomes: dict[str, CaseOutcomeSummary] = Field(default_factory=dict) case_errors: list[dict[str, str]] = Field(default_factory=list) model_configs: dict[str, Any] = Field(default_factory=dict) diff --git a/tests/unit/test_cost_tracking.py b/tests/unit/test_cost_tracking.py index 10676c0..a9cecfa 100644 --- a/tests/unit/test_cost_tracking.py +++ b/tests/unit/test_cost_tracking.py @@ -1,21 +1,18 @@ """Tests for cost tracking module.""" -import pytest - from agenteval.evaluation.cost_tracking import ( + DEFAULT_PRICING, CostBreakdown, ModelPricing, - TokenUsage, - aggregate_token_usage, - calculate_case_cost, + build_eval_cost_section, calculate_cost, - calculate_run_cost, - calculate_turn_cost, + get_pricing, + reload_pricing_overrides, + usage_breakdown, ) def test_model_pricing_model(): - """ModelPricing model should work correctly.""" pricing = ModelPricing( model_id="gpt-4o", prompt_cost_per_1m=5.0, @@ -25,117 +22,100 @@ def test_model_pricing_model(): assert pricing.prompt_cost_per_1m == 5.0 -def test_token_usage_model(): - """TokenUsage model should work correctly.""" - usage = TokenUsage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - assert usage.prompt_tokens == 100 - assert usage.completion_tokens == 50 - assert usage.total_tokens == 150 - - def test_calculate_cost_gpt4o_mini(): - """Cost calculation for gpt-4o-mini should be correct.""" pricing = ModelPricing(model_id="gpt-4o-mini", prompt_cost_per_1m=0.15, completion_cost_per_1m=0.60) - # 1000 prompt tokens + 500 completion tokens - # Cost = (1000/1M * 0.15) + (500/1M * 0.60) = 0.00015 + 0.0003 = 0.00045 + # (1000/1M * 0.15) + (500/1M * 0.60) = 0.00015 + 0.0003 = 0.00045 cost = calculate_cost(1000, 500, pricing) assert abs(cost - 0.00045) < 0.00001 def test_calculate_cost_zero_tokens(): - """Zero tokens should result in zero cost.""" pricing = ModelPricing(model_id="gpt-4o", prompt_cost_per_1m=5.0, completion_cost_per_1m=15.0) - cost = calculate_cost(0, 0, pricing) - assert cost == 0.0 + assert calculate_cost(0, 0, pricing) == 0.0 -def test_aggregate_token_usage(): - """Aggregate token usage from multiple turns.""" - turns = [ - {"prompt_tokens": 100, "completion_tokens": 50}, - {"prompt_tokens": 200, "completion_tokens": 100}, - {"prompt_tokens": None, "completion_tokens": None}, # Missing data - ] - usage = aggregate_token_usage(turns) - assert usage.prompt_tokens == 300 - assert usage.completion_tokens == 150 - assert usage.total_tokens == 450 +def test_get_pricing_known_model(): + pricing = get_pricing("gpt-4o-mini") + assert pricing is not None + assert pricing.prompt_cost_per_1m == DEFAULT_PRICING["gpt-4o-mini"].prompt_cost_per_1m -def test_aggregate_token_usage_empty(): - """Empty turns should result in zero usage.""" - usage = aggregate_token_usage([]) - assert usage.prompt_tokens == 0 - assert usage.completion_tokens == 0 - assert usage.total_tokens == 0 +def test_get_pricing_unknown_or_empty(): + assert get_pricing("no-such-model") is None + assert get_pricing(None) is None + assert get_pricing("") is None -def test_calculate_turn_cost(): - """Calculate cost for a single turn.""" - pricing = ModelPricing(model_id="gpt-4o-mini", prompt_cost_per_1m=0.15, completion_cost_per_1m=0.60) - turn = {"prompt_tokens": 1000, "completion_tokens": 500} - breakdown = calculate_turn_cost(turn, pricing) - assert breakdown.prompt_tokens == 1000 - assert breakdown.completion_tokens == 500 +def test_get_pricing_override_file(tmp_path, monkeypatch): + override_file = tmp_path / "model_pricing.json" + override_file.write_text( + '{"custom-judge": {"prompt_cost_per_1m": 1.0, "completion_cost_per_1m": 2.0},' + ' "broken": "not-a-dict"}', + encoding="utf-8", + ) + monkeypatch.setattr("agenteval.evaluation.cost_tracking._PRICING_FILE", override_file) + reload_pricing_overrides() + try: + pricing = get_pricing("custom-judge") + assert pricing is not None + assert pricing.completion_cost_per_1m == 2.0 + # 覆盖优先于默认表 + override_file.write_text( + '{"gpt-4o-mini": {"prompt_cost_per_1m": 9.9, "completion_cost_per_1m": 9.9}}', + encoding="utf-8", + ) + reload_pricing_overrides() + assert get_pricing("gpt-4o-mini").prompt_cost_per_1m == 9.9 + # 坏条目被跳过,不抛异常 + assert get_pricing("broken") is None + finally: + reload_pricing_overrides() + + +def test_get_pricing_missing_file_is_noop(tmp_path, monkeypatch): + monkeypatch.setattr("agenteval.evaluation.cost_tracking._PRICING_FILE", tmp_path / "missing.json") + reload_pricing_overrides() + try: + assert get_pricing("gpt-4o") is DEFAULT_PRICING["gpt-4o"] + finally: + reload_pricing_overrides() + + +def test_usage_breakdown(): + pricing = DEFAULT_PRICING["gpt-4o-mini"] + breakdown = usage_breakdown({"prompt_tokens": 1000, "completion_tokens": 500}, pricing) + assert isinstance(breakdown, CostBreakdown) assert breakdown.total_tokens == 1500 assert abs(breakdown.cost_usd - 0.00045) < 0.00001 -def test_calculate_turn_cost_missing_tokens(): - """Turn with missing token data should have zero cost.""" - pricing = ModelPricing(model_id="gpt-4o", prompt_cost_per_1m=5.0, completion_cost_per_1m=15.0) - turn = {} # No token data - breakdown = calculate_turn_cost(turn, pricing) - assert breakdown.prompt_tokens == 0 - assert breakdown.completion_tokens == 0 +def test_usage_breakdown_missing_fields(): + pricing = DEFAULT_PRICING["gpt-4o-mini"] + breakdown = usage_breakdown({}, pricing) + assert breakdown.total_tokens == 0 assert breakdown.cost_usd == 0.0 -def test_calculate_case_cost(): - """Calculate cost for a case with multiple turns.""" - pricing = ModelPricing(model_id="gpt-4o-mini", prompt_cost_per_1m=0.15, completion_cost_per_1m=0.60) - turns = [ - {"prompt_tokens": 1000, "completion_tokens": 500}, - {"prompt_tokens": 2000, "completion_tokens": 1000}, - ] - breakdown = calculate_case_cost(turns, pricing) - assert breakdown.prompt_tokens == 3000 - assert breakdown.completion_tokens == 1500 - assert breakdown.total_tokens == 4500 - # Cost = (3000/1M * 0.15) + (1500/1M * 0.60) = 0.00045 + 0.0009 = 0.00135 - assert abs(breakdown.cost_usd - 0.00135) < 0.00001 +def test_build_eval_cost_section_by_purpose(): + usage = { + "judge": {"prompt_tokens": 1000, "completion_tokens": 500, "total_tokens": 1500}, + "generator": {"prompt_tokens": 2000, "completion_tokens": 1000, "total_tokens": 3000}, + } + model_configs = { + "judge": {"model_name": "gpt-4o-mini"}, + "generator": {"model_name": "unknown-model"}, + } + section = build_eval_cost_section(usage, model_configs) + assert section is not None + by_purpose = {i["purpose"]: i for i in section["by_purpose"]} + assert abs(by_purpose["judge"]["cost_usd"] - 0.00045) < 0.00001 + # 未知定价:token 数仍展示,成本为空 + assert by_purpose["generator"]["cost_usd"] is None + assert by_purpose["generator"]["total_tokens"] == 3000 + assert section["total_tokens"] == 4500 + assert abs(section["total_cost_usd"] - 0.00045) < 0.00001 -def test_calculate_run_cost(): - """Calculate total cost for a run with multiple cases.""" - pricing = ModelPricing(model_id="gpt-4o-mini", prompt_cost_per_1m=0.15, completion_cost_per_1m=0.60) - cases = [ - { - "case_id": "c1", - "turns": [ - {"prompt_tokens": 1000, "completion_tokens": 500}, - ], - }, - { - "case_id": "c2", - "turns": [ - {"prompt_tokens": 2000, "completion_tokens": 1000}, - ], - }, - ] - breakdown = calculate_run_cost(cases, pricing) - assert breakdown.prompt_tokens == 3000 - assert breakdown.completion_tokens == 1500 - assert breakdown.total_tokens == 4500 - - -def test_cost_breakdown_model(): - """CostBreakdown model should work correctly.""" - breakdown = CostBreakdown( - prompt_tokens=1000, - completion_tokens=500, - total_tokens=1500, - cost_usd=0.001, - ) - assert breakdown.prompt_tokens == 1000 - assert breakdown.cost_usd == 0.001 +def test_build_eval_cost_section_empty(): + assert build_eval_cost_section(None, None) is None + assert build_eval_cost_section({}, {}) is None diff --git a/tests/unit/test_llm_score.py b/tests/unit/test_llm_score.py index 4453595..8558e65 100644 --- a/tests/unit/test_llm_score.py +++ b/tests/unit/test_llm_score.py @@ -68,7 +68,7 @@ async def test_llm_score_passes_above_threshold(): rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6}) mock_resp = _make_llm_response(score=8.0, reason="很好") - with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: instance = MockClient.return_value.__aenter__.return_value instance.post = AsyncMock(return_value=mock_resp) result = await rule.evaluate(_case(), [_turn("优质回答")]) @@ -82,7 +82,7 @@ async def test_llm_score_fails_below_threshold(): rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 7}) mock_resp = _make_llm_response(score=4.0, reason="较差") - with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: instance = MockClient.return_value.__aenter__.return_value instance.post = AsyncMock(return_value=mock_resp) result = await rule.evaluate(_case(), [_turn("差劲回答")]) @@ -96,7 +96,7 @@ async def test_llm_score_clamps_score_to_0_10(): # API returns out-of-range score mock_resp = _make_llm_response(score=12.0) - with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: instance = MockClient.return_value.__aenter__.return_value instance.post = AsyncMock(return_value=mock_resp) result = await rule.evaluate(_case(), [_turn("answer")]) @@ -110,7 +110,7 @@ async def test_llm_score_handles_content_block_array(): rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6}) mock_resp = _make_content_block_response(score=7.5) - with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: instance = MockClient.return_value.__aenter__.return_value instance.post = AsyncMock(return_value=mock_resp) result = await rule.evaluate(_case(), [_turn("answer")]) @@ -129,7 +129,7 @@ async def test_llm_score_parses_json_with_preamble(): "choices": [{"message": {"content": 'Sure! Here is the result: {"score": 7, "reason": "decent"}'}}] }) - with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: instance = MockClient.return_value.__aenter__.return_value instance.post = AsyncMock(return_value=mock_resp) result = await rule.evaluate(_case(), [_turn("answer")]) @@ -142,7 +142,7 @@ async def test_llm_score_parses_json_with_preamble(): async def test_llm_score_api_error_fails_gracefully(): rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6}) - with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: instance = MockClient.return_value.__aenter__.return_value instance.post = AsyncMock(side_effect=Exception("connection timeout")) result = await rule.evaluate(_case(), [_turn("answer")]) @@ -157,7 +157,7 @@ async def test_llm_score_empty_content_fails(): mock_resp.raise_for_status = MagicMock() mock_resp.json = MagicMock(return_value={"choices": []}) - with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: instance = MockClient.return_value.__aenter__.return_value instance.post = AsyncMock(return_value=mock_resp) result = await rule.evaluate(_case(), [_turn("answer")]) @@ -178,7 +178,7 @@ async def test_llm_score_extracts_question_from_sent_message(): captured_payload.update(kwargs.get("json", {})) return mock_resp - with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: instance = MockClient.return_value.__aenter__.return_value instance.post = AsyncMock(side_effect=capture_post) await rule.evaluate(_case(), [_turn("答案内容", sent_text="这是用户的问题")]) @@ -208,7 +208,7 @@ async def test_llm_score_multiturn_uses_last_sent_not_prev_reply(): _turn("第三轮AI回复", sent_text="第三轮用户问题"), ] - with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: instance = MockClient.return_value.__aenter__.return_value instance.post = AsyncMock(side_effect=capture_post) await rule.evaluate(_case(), dialog) @@ -227,7 +227,7 @@ async def test_llm_score_reason_includes_llm_detail(): rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5}) mock_resp = _make_llm_response(score=3.0, reason="回复偏离主题") - with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: instance = MockClient.return_value.__aenter__.return_value instance.post = AsyncMock(return_value=mock_resp) result = await rule.evaluate(_case(), [_turn("answer")]) diff --git a/tests/unit/test_model_runtime_integration.py b/tests/unit/test_model_runtime_integration.py index db2ac83..7edee84 100644 --- a/tests/unit/test_model_runtime_integration.py +++ b/tests/unit/test_model_runtime_integration.py @@ -12,10 +12,14 @@ class FakeGateway: self.chat_calls = 0 self.total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} - async def chat(self, config, messages, temperature=0.2): + async def chat_with_usage(self, config, messages, temperature=0.2): self.chat_calls += 1 assert config.name == "生成模型" - return '["问题一", "问题二"]' + return '["问题一", "问题二"]', {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} + + async def chat(self, config, messages, temperature=0.2): + content, _ = await self.chat_with_usage(config, messages, temperature) + return content async def test_dynamic_case_uses_generator_binding_and_records_snapshot(db_session): @@ -62,3 +66,5 @@ async def test_dynamic_case_uses_generator_binding_and_records_snapshot(db_sessi assert snapshot["id"] == config_id assert snapshot["model_name"] == "generator-v1" assert "api_key" not in snapshot + # 生成岗位的用量按次归集进 summary(分岗位成本核算的数据源) + assert run.summary.eval_usage_by_purpose["generator"]["total_tokens"] == 15 diff --git a/tests/unit/test_phase3_wiring.py b/tests/unit/test_phase3_wiring.py new file mode 100644 index 0000000..e7d06a1 --- /dev/null +++ b/tests/unit/test_phase3_wiring.py @@ -0,0 +1,163 @@ +"""Phase 3 (v1.3.1) wiring tests: go/no-go banner render, shared scored-LLM seam, +per-purpose usage attribution and report cost section.""" + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from agenteval.evaluation.report_render import render_html, render_markdown +from agenteval.evaluation.rules.llm_score import LlmScoreRule +from agenteval.models import Case, CaseType, Turn + +from tests.unit.test_phase2_wiring import report_seeded # noqa: F401 + +# ── 3.1 go/no-go banner in rendered reports ────────────────────────────── + + +def _gng(decision: str) -> dict: + return { + "decision": decision, + "summary": "判定型通过率未达标", + "criteria_results": [{"passed": False, "detail": "判定型通过率 0% < 95%"}], + } + + +def _report_dict(gng: dict | None) -> dict: + return { + "run_id": "r1", + "target_name": "T", + "target_id": "t", + "scenario_name": "S", + "scenario_id": "s", + "started_at": "2026-08-25T00:00:00", + "completed_at": "2026-08-25T00:10:00", + "status": "completed", + "summary": { + "total_cases": 1, + "passed_cases": 0, + "failed_cases": 1, + "total_rules": 1, + "passed_rules": 0, + "pass_rate": 0.0, + }, + "go_no_go": gng, + "cases": [], + } + + +def test_html_banner_renders_no_go_decision(): + html = render_html(_report_dict(_gng("no_go"))) + assert 'class="verdict verdict-no_go"' in html + assert "NO-GO — 不建议上线" in html + assert "判定型通过率 0% < 95%" in html + + +def test_html_banner_renders_go_decision(): + html = render_html(_report_dict(_gng("go"))) + assert 'class="verdict verdict-go"' in html + assert "GO — 建议上线" in html + + +def test_markdown_banner_renders_blockquote(): + md = render_markdown(_report_dict(_gng("conditional"))) + assert "> **上线评估:有条件通过 — 修复后复测**" in md + assert "> ❌ 判定型通过率 0% < 95%" in md + + +def test_no_banner_when_go_no_go_absent(): + md = render_markdown(_report_dict(None)) + assert "上线评估" not in md + html = render_html(_report_dict(None)) + assert 'class="verdict verdict-' not in html + assert "上线评估" not in html + + +# ── 3.2 rule-level usage recording via chat_with_usage ────────────────── + + +@pytest.mark.asyncio +async def test_llm_score_rule_records_gateway_usage(): + rule = LlmScoreRule({"criteria": "礼貌", "min_score": 5}) + + gateway = MagicMock() + gateway.chat_with_usage = AsyncMock( + return_value=(json.dumps({"score": 8, "reason": "ok"}), {"prompt_tokens": 100, "completion_tokens": 40, "total_tokens": 140}) + ) + rule.gateway = gateway + rule.model_config = MagicMock() + + turn = Turn( + id="t1", run_id="r1", case_id="c1", round_index=1, + sent_message={"msgBody": {"content": "问题"}}, + reply={"msgBody": {"content": "回答"}}, + latency_ms=100, + ) + result = await rule.evaluate(Case(id="c1", type=CaseType.SINGLE, messages=["x"]), [turn]) + assert result.passed is True + assert rule.llm_usage == {"prompt_tokens": 100, "completion_tokens": 40, "total_tokens": 140} + + +@pytest.mark.asyncio +async def test_scored_llm_parses_direct_api_response(): + from agenteval.evaluation.rules.scored_llm import call_scored_llm + + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.json = MagicMock(return_value={ + "choices": [{"message": {"content": json.dumps({"score": 9, "reason": "好"})}}] + }) + with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(return_value=mock_resp) + score, reason = await call_scored_llm("http://mock", None, "m", "sys", "user") + assert score == 9 + assert reason == "好" + + +# ── 3.3 report cost section from recorded per-purpose usage ───────────── + + +@pytest.mark.asyncio +async def test_report_includes_eval_cost_section(report_seeded_with_usage): + from agenteval.evaluation.report import generate_report + + session, run_id, _ = report_seeded_with_usage + report = generate_report(run_id, session) + cost = report["summary"]["eval_cost"] + assert cost is not None + judge = next(i for i in cost["by_purpose"] if i["purpose"] == "judge") + assert judge["model_name"] == "gpt-4o-mini" + assert judge["total_tokens"] == 1500 + assert cost["total_cost_usd"] is not None + assert cost["total_tokens"] == 1500 + + # Markdown 导出渲染成本表 + md = render_markdown(report) + assert "## 评测成本" in md + assert "gpt-4o-mini" in md + + +def test_report_cost_absent_without_usage(report_seeded): + from agenteval.evaluation.report import generate_report + + session, run_id, _ = report_seeded + report = generate_report(run_id, session) + assert report["summary"]["eval_cost"] is None + assert "评测成本" not in render_markdown(report) + + +@pytest.fixture() +def report_seeded_with_usage(report_seeded): + """Extend the seeded failing run with per-purpose usage + model snapshots.""" + from agenteval.storage.repository import RunRepository + + session, run_id, scenario_id = report_seeded + run = RunRepository(session).get(run_id) + summary = dict(run.summary.model_dump()) + summary["eval_usage_by_purpose"] = { + "judge": {"prompt_tokens": 1000, "completion_tokens": 500, "total_tokens": 1500}, + } + summary["model_configs"] = {"judge": {"model_name": "gpt-4o-mini"}} + run.summary = summary + RunRepository(session).update(run) + return session, run_id, scenario_id