diff --git a/backend/agenteval/evaluation/cost_tracking.py b/backend/agenteval/evaluation/cost_tracking.py index ea41a44..5297581 100644 --- a/backend/agenteval/evaluation/cost_tracking.py +++ b/backend/agenteval/evaluation/cost_tracking.py @@ -1,9 +1,21 @@ -"""Cost tracking and calculation for evaluation runs.""" +"""Cost tracking for evaluation runs — pricing lookup and usage-based cost. -from typing import Any +成本口径:评测自身消耗的 LLM token(judge/generator/embedding/moderation, +经 ModelGateway 归集进 RunSummary.eval_usage_by_purpose),按岗位所用模型的 +单价计费。被评智能体的通道用量不在口径内(tutu-api 通道不返回用量)。 + +单价解析顺序:``data/model_pricing.json``(部署侧覆盖)→ ``DEFAULT_PRICING``。 +""" + +import json +from pathlib import Path +from typing import Any, Optional from pydantic import BaseModel, Field +# data/model_pricing.json:与 DB 同目录,部署时挂载覆盖,缺省文件视为无覆盖 +_PRICING_FILE = Path(__file__).resolve().parents[3] / "data" / "model_pricing.json" + class ModelPricing(BaseModel): """Pricing for a model (per 1M tokens).""" @@ -13,16 +25,8 @@ class ModelPricing(BaseModel): completion_cost_per_1m: float = Field(description="Cost per 1M completion tokens in USD") -class TokenUsage(BaseModel): - """Token usage for a single API call.""" - - prompt_tokens: int = 0 - completion_tokens: int = 0 - total_tokens: int = 0 - - class CostBreakdown(BaseModel): - """Cost breakdown for a turn, case, or run.""" + """Cost breakdown for one usage bucket (e.g. one purpose or a whole run).""" prompt_tokens: int = 0 completion_tokens: int = 0 @@ -40,118 +44,99 @@ DEFAULT_PRICING: dict[str, ModelPricing] = { "claude-3-haiku": ModelPricing(model_id="claude-3-haiku", prompt_cost_per_1m=0.25, completion_cost_per_1m=1.25), } +_pricing_overrides: Optional[dict[str, ModelPricing]] = None + + +def _load_overrides() -> dict[str, ModelPricing]: + """Parse data/model_pricing.json once; a missing/broken file means no override.""" + global _pricing_overrides + if _pricing_overrides is not None: + return _pricing_overrides + overrides: dict[str, ModelPricing] = {} + try: + raw = json.loads(_PRICING_FILE.read_text(encoding="utf-8")) + if isinstance(raw, dict): + for name, entry in raw.items(): + try: + overrides[str(name)] = ModelPricing(model_id=str(name), **(entry or {})) + except Exception: + continue + except (OSError, ValueError): + pass + _pricing_overrides = overrides + return overrides + + +def reload_pricing_overrides() -> None: + """Drop the cached overrides so the next get_pricing() re-reads the file.""" + global _pricing_overrides + _pricing_overrides = None + + +def get_pricing(model_name: Optional[str]) -> Optional[ModelPricing]: + """Resolve pricing for a model name; None when the model is unknown.""" + if not model_name: + return None + return _load_overrides().get(model_name) or DEFAULT_PRICING.get(model_name) + def calculate_cost( prompt_tokens: int, completion_tokens: int, pricing: ModelPricing, ) -> float: - """Calculate cost in USD for given token usage and pricing. - - Args: - prompt_tokens: Number of prompt tokens - completion_tokens: Number of completion tokens - pricing: Model pricing configuration - - Returns: - Cost in USD - """ + """Calculate cost in USD for given token usage and pricing.""" prompt_cost = (prompt_tokens / 1_000_000) * pricing.prompt_cost_per_1m completion_cost = (completion_tokens / 1_000_000) * pricing.completion_cost_per_1m return prompt_cost + completion_cost -def aggregate_token_usage(turns: list[dict[str, Any]]) -> TokenUsage: - """Aggregate token usage from a list of turns. - - Args: - turns: List of turn dicts with optional token fields - - Returns: - Aggregated token usage - """ - prompt_tokens = sum(t.get("prompt_tokens") or 0 for t in turns) - completion_tokens = sum(t.get("completion_tokens") or 0 for t in turns) - return TokenUsage( +def usage_breakdown(usage: dict[str, Any], pricing: ModelPricing) -> CostBreakdown: + """Turn one recorded usage dict (prompt/completion tokens) into a CostBreakdown.""" + prompt_tokens = int(usage.get("prompt_tokens") or 0) + completion_tokens = int(usage.get("completion_tokens") or 0) + return CostBreakdown( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, + cost_usd=calculate_cost(prompt_tokens, completion_tokens, pricing), ) -def calculate_turn_cost( - turn: dict[str, Any], - pricing: ModelPricing, -) -> CostBreakdown: - """Calculate cost for a single turn. +def build_eval_cost_section( + usage_by_purpose: Optional[dict[str, dict[str, Any]]], + model_configs: Optional[dict[str, Any]], +) -> Optional[dict[str, Any]]: + """Per-purpose eval-LLM cost from recorded run usage; None when no usage. - Args: - turn: Turn dict with optional token fields - pricing: Model pricing configuration - - Returns: - Cost breakdown for the turn + ``model_configs`` is the run summary's purpose → model snapshot map; a purpose + whose model has no known pricing gets ``cost_usd: None`` (tokens still shown). """ - prompt_tokens = turn.get("prompt_tokens") or 0 - completion_tokens = turn.get("completion_tokens") or 0 - total_tokens = prompt_tokens + completion_tokens - cost_usd = calculate_cost(prompt_tokens, completion_tokens, pricing) - - return CostBreakdown( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - cost_usd=cost_usd, - ) - - -def calculate_case_cost( - turns: list[dict[str, Any]], - pricing: ModelPricing, -) -> CostBreakdown: - """Calculate cost for a case (multiple turns). - - Args: - turns: List of turn dicts - pricing: Model pricing configuration - - Returns: - Aggregated cost breakdown for the case - """ - usage = aggregate_token_usage(turns) - cost_usd = calculate_cost(usage.prompt_tokens, usage.completion_tokens, pricing) - - return CostBreakdown( - prompt_tokens=usage.prompt_tokens, - completion_tokens=usage.completion_tokens, - total_tokens=usage.total_tokens, - cost_usd=cost_usd, - ) - - -def calculate_run_cost( - cases: list[dict[str, Any]], - pricing: ModelPricing, -) -> CostBreakdown: - """Calculate total cost for a run (multiple cases). - - Args: - cases: List of case dicts, each with a 'turns' field - pricing: Model pricing configuration - - Returns: - Aggregated cost breakdown for the run - """ - all_turns = [] - for case in cases: - all_turns.extend(case.get("turns", [])) - - usage = aggregate_token_usage(all_turns) - cost_usd = calculate_cost(usage.prompt_tokens, usage.completion_tokens, pricing) - - return CostBreakdown( - prompt_tokens=usage.prompt_tokens, - completion_tokens=usage.completion_tokens, - total_tokens=usage.total_tokens, - cost_usd=cost_usd, - ) + if not usage_by_purpose: + return None + model_configs = model_configs or {} + items: list[dict[str, Any]] = [] + total_cost = 0.0 + has_any_cost = False + for purpose, usage in sorted(usage_by_purpose.items()): + model_name = (model_configs.get(purpose) or {}).get("model_name") + pricing = get_pricing(model_name) + breakdown = usage_breakdown(usage, pricing) if pricing else None + if breakdown is not None: + total_cost += breakdown.cost_usd + has_any_cost = True + items.append( + { + "purpose": purpose, + "model_name": model_name, + "prompt_tokens": breakdown.prompt_tokens if breakdown else int(usage.get("prompt_tokens") or 0), + "completion_tokens": breakdown.completion_tokens if breakdown else int(usage.get("completion_tokens") or 0), + "total_tokens": breakdown.total_tokens if breakdown else int(usage.get("total_tokens") or 0), + "cost_usd": breakdown.cost_usd if breakdown else None, + } + ) + return { + "by_purpose": items, + "total_tokens": sum(i["total_tokens"] for i in items), + "total_cost_usd": round(total_cost, 6) if has_any_cost else None, + } diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index 373af8f..d03acf3 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -118,6 +118,9 @@ class EvalEngine: max(1, get_settings().max_concurrent_rules), ) self._state_lock = asyncio.Lock() + # 分岗位的评测侧 token 用量(ModelPurpose.value → 用量),由各规则/生成器 + # 的按次用量归集而来,避免并发下用网关总量做差产生竞态。 + self._usage_by_purpose: dict[str, dict[str, int]] = {} # Collects fatal case-level errors (e.g. dynamic message generation # failures) so their cause is persisted into run.summary — not just # emitted transiently over WebSocket. @@ -216,6 +219,7 @@ class EvalEngine: case_errors=self._case_errors or None, model_configs=resolved_snapshot or None, eval_token_usage=usage if usage["total_tokens"] > 0 else None, + eval_usage_by_purpose=self._usage_by_purpose or None, ) run.status = RunStatus.COMPLETED run.completed_at = utc_now() @@ -479,6 +483,8 @@ class EvalEngine: gateway=self.model_gateway if model_config else None, ) result = await rule.evaluate(case, dialog) + if purpose: + await self._accumulate_usage(purpose.value, rule.llm_usage) except Exception as exc: result = RuleResult(passed=False, reason=f"模型配置解析失败: {exc}") return rule_config, is_implicit, result @@ -571,7 +577,10 @@ class EvalEngine: try: model_config = await self._resolve_model(ModelPurpose.GENERATOR) if model_config: - content = await self.model_gateway.chat(model_config, messages_payload, temperature=0.7) + content, usage = await self.model_gateway.chat_with_usage( + model_config, messages_payload, temperature=0.7 + ) + await self._accumulate_usage(ModelPurpose.GENERATOR.value, usage) else: content = await self._generate_messages_legacy(messages_payload) @@ -625,6 +634,16 @@ class EvalEngine: raise ValueError("LLM 返回内容为空或无法解析") return content + async def _accumulate_usage(self, purpose_key: str, usage: dict[str, int] | None) -> None: + if not usage or not usage.get("total_tokens"): + return + async with self._state_lock: + bucket = self._usage_by_purpose.setdefault( + purpose_key, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + ) + for key in bucket: + bucket[key] += int(usage.get(key) or 0) + async def _resolve_model(self, purpose: ModelPurpose | None) -> ModelRuntimeConfig | None: if purpose is None: return None diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index 1786490..5016eae 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -12,6 +12,7 @@ from typing import Any, Optional from sqlmodel import Session from agenteval.evaluation.case_verdict import build_case_evidence, resolve_case_verdicts +from agenteval.evaluation.cost_tracking import build_eval_cost_section from agenteval.evaluation.go_no_go import AcceptanceCriteria, evaluate_go_no_go from agenteval.evaluation.metrics import aggregate_runs from agenteval.evaluation.report_render import render_html, render_json, render_markdown @@ -117,6 +118,8 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: "judged_pass_rate": judged_pass_rate, "avg_latency_ms": summary.avg_latency_ms, "eval_token_usage": summary.eval_token_usage, + "eval_usage_by_purpose": summary.eval_usage_by_purpose, + "eval_cost": build_eval_cost_section(summary.eval_usage_by_purpose, summary.model_configs), } # Generate go/no-go verdict(场景级验收标准优先,缺省用全局默认) diff --git a/backend/agenteval/evaluation/report_render.py b/backend/agenteval/evaluation/report_render.py index 5c5616b..7120d4b 100644 --- a/backend/agenteval/evaluation/report_render.py +++ b/backend/agenteval/evaluation/report_render.py @@ -33,6 +33,12 @@ HTML_TEMPLATE = """ .badge { padding: 2px 8px; border-radius: 4px; font-size: 12px; } .pass { background: #e6f7e6; color: #2e7d32; } .fail { background: #ffebee; color: #c62828; } + .verdict { border-radius: 6px; padding: 16px; margin: 24px 0; } + .verdict .decision { font-size: 18px; font-weight: 700; } + .verdict ul { margin: 8px 0 0; padding-left: 20px; } + .verdict-go { background: #e6f7e6; border: 1px solid #b7e0b8; color: #2e7d32; } + .verdict-no_go { background: #ffebee; border: 1px solid #f5c6cb; color: #c62828; } + .verdict-conditional { background: #fff8e1; border: 1px solid #ffe082; color: #8d6e00; } pre { white-space: pre-wrap; word-break: break-word; background: #f5f5f5; padding: 8px; border-radius: 4px; } @@ -43,6 +49,22 @@ HTML_TEMPLATE = """
评测场景:{{ report.scenario_name }}({{ report.scenario_id }})
执行时间:{{ report.started_at }} 至 {{ report.completed_at or '进行中' }}
+ {% if report.go_no_go %} + {% set gng = report.go_no_go %} + {% set decision_label = {'go': 'GO — 建议上线', 'no_go': 'NO-GO — 不建议上线', 'conditional': '有条件通过 — 修复后复测'} %} +