2f09ee2bfc
All checks were successful
CI / test (pull_request) Successful in 3m58s
- 报告渲染 Go/No-Go 上线评估横幅（HTML 彩色 banner + Markdown 引用块） - 抽取 scored_llm 共享模块：llm_score / fluency 直连调用与评分解析收敛 - 网关新增 chat_with_usage / embed_with_usage，规则按次归集 llm_usage - 引擎分岗位用量归集（judge/generator/embedding/moderation）写入 RunSummary.eval_usage_by_purpose，并发下不做总量差值 - cost_tracking 重构：data/model_pricing.json 覆盖 + 默认计价表， 删除从未有数据支撑的 Turn 维度成本函数（偏差说明见 PR） - 报告 summary 增加 eval_cost 分岗位成本段并在 Markdown 渲染
64 lines
2.6 KiB
Python
64 lines
2.6 KiB
Python
"""Single-run summary aggregation — the pure落点 for one run's口径.
|
|
|
|
Parallel to ``metrics.aggregate_runs`` (cross-run) and
|
|
``judgement.combine_case_outcome`` (case-level): given the authoritative
|
|
per-case outcomes plus raw latency/rule material, compute the run's
|
|
summary口径 once. No IO — the engine collects material and calls this; DB
|
|
writes and event emits stay in the caller. See CONTEXT.md (通过率) / ADR-0002.
|
|
"""
|
|
|
|
from typing import Any, Optional, Sequence
|
|
|
|
from agenteval.evaluation.judgement import CaseOutcome
|
|
from agenteval.models import CaseOutcomeSummary, RunSummary
|
|
|
|
|
|
def build_run_summary(
|
|
*,
|
|
case_outcomes: dict[str, CaseOutcome],
|
|
latencies: Sequence[float],
|
|
rule_passes: Sequence[bool],
|
|
case_errors: Optional[list[dict[str, str]]] = None,
|
|
model_configs: Optional[dict[str, Any]] = None,
|
|
eval_token_usage: Optional[dict[str, int]] = None,
|
|
eval_usage_by_purpose: Optional[dict[str, dict[str, int]]] = None,
|
|
) -> RunSummary:
|
|
"""Compute a run's summary口径 from its authoritative case outcomes."""
|
|
total_cases = len(case_outcomes)
|
|
passed_cases = sum(1 for o in case_outcomes.values() if o.passed)
|
|
connectivity_count = sum(1 for o in case_outcomes.values() if o.connectivity)
|
|
abandoned_cases = sum(1 for o in case_outcomes.values() if o.abandoned)
|
|
|
|
pass_rate = round(passed_cases / total_cases, 4) if total_cases else 0.0
|
|
abandonment_rate = round(abandoned_cases / total_cases, 4) if total_cases else None
|
|
# 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数
|
|
judged_total = total_cases - connectivity_count
|
|
judged_pass_rate = (
|
|
round((passed_cases - connectivity_count) / judged_total, 4) if judged_total > 0 else None
|
|
)
|
|
|
|
avg_latency_ms = round(sum(latencies) / len(latencies), 1) if latencies else None
|
|
|
|
return RunSummary(
|
|
total_cases=total_cases,
|
|
passed_cases=passed_cases,
|
|
failed_cases=total_cases - passed_cases,
|
|
abandoned_cases=abandoned_cases,
|
|
total_rules=len(rule_passes),
|
|
passed_rules=sum(1 for p in rule_passes if p),
|
|
pass_rate=pass_rate,
|
|
judged_pass_rate=judged_pass_rate,
|
|
abandonment_rate=abandonment_rate,
|
|
avg_latency_ms=avg_latency_ms,
|
|
eval_token_usage=eval_token_usage,
|
|
eval_usage_by_purpose=eval_usage_by_purpose,
|
|
case_outcomes={
|
|
case_id: CaseOutcomeSummary(
|
|
passed=o.passed, connectivity=o.connectivity, abandoned=o.abandoned
|
|
)
|
|
for case_id, o in case_outcomes.items()
|
|
},
|
|
case_errors=case_errors or [],
|
|
model_configs=model_configs or {},
|
|
)
|