c956da7686
All checks were successful
CI / test (pull_request) Successful in 4m3s
- token 用量接入：ModelGateway 经 adapter.parse_usage 累计评测侧 LLM 调用的 token 消耗，引擎写入 run.summary.eval_token_usage，报告透出 - 放弃率落地：CaseOutcome 新增 abandoned 标记（对话中途发送/接收失败）， build_run_summary 统计 abandoned_cases / abandonment_rate - Go/No-Go 可配置：Scenario 新增 acceptance_criteria 字段（DB 列 + 幂等迁移）， 报告按场景标准出 verdict，缺省回退全局默认；标准变更不触发考纲升版
62 lines
2.5 KiB
Python
62 lines
2.5 KiB
Python
"""Single-run summary aggregation — the pure落点 for one run's口径.
|
|
|
|
Parallel to ``metrics.aggregate_runs`` (cross-run) and
|
|
``judgement.combine_case_outcome`` (case-level): given the authoritative
|
|
per-case outcomes plus raw latency/rule material, compute the run's
|
|
summary口径 once. No IO — the engine collects material and calls this; DB
|
|
writes and event emits stay in the caller. See CONTEXT.md (通过率) / ADR-0002.
|
|
"""
|
|
|
|
from typing import Any, Optional, Sequence
|
|
|
|
from agenteval.evaluation.judgement import CaseOutcome
|
|
from agenteval.models import CaseOutcomeSummary, RunSummary
|
|
|
|
|
|
def build_run_summary(
|
|
*,
|
|
case_outcomes: dict[str, CaseOutcome],
|
|
latencies: Sequence[float],
|
|
rule_passes: Sequence[bool],
|
|
case_errors: Optional[list[dict[str, str]]] = None,
|
|
model_configs: Optional[dict[str, Any]] = None,
|
|
eval_token_usage: Optional[dict[str, int]] = None,
|
|
) -> RunSummary:
|
|
"""Compute a run's summary口径 from its authoritative case outcomes."""
|
|
total_cases = len(case_outcomes)
|
|
passed_cases = sum(1 for o in case_outcomes.values() if o.passed)
|
|
connectivity_count = sum(1 for o in case_outcomes.values() if o.connectivity)
|
|
abandoned_cases = sum(1 for o in case_outcomes.values() if o.abandoned)
|
|
|
|
pass_rate = round(passed_cases / total_cases, 4) if total_cases else 0.0
|
|
abandonment_rate = round(abandoned_cases / total_cases, 4) if total_cases else None
|
|
# 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数
|
|
judged_total = total_cases - connectivity_count
|
|
judged_pass_rate = (
|
|
round((passed_cases - connectivity_count) / judged_total, 4) if judged_total > 0 else None
|
|
)
|
|
|
|
avg_latency_ms = round(sum(latencies) / len(latencies), 1) if latencies else None
|
|
|
|
return RunSummary(
|
|
total_cases=total_cases,
|
|
passed_cases=passed_cases,
|
|
failed_cases=total_cases - passed_cases,
|
|
abandoned_cases=abandoned_cases,
|
|
total_rules=len(rule_passes),
|
|
passed_rules=sum(1 for p in rule_passes if p),
|
|
pass_rate=pass_rate,
|
|
judged_pass_rate=judged_pass_rate,
|
|
abandonment_rate=abandonment_rate,
|
|
avg_latency_ms=avg_latency_ms,
|
|
eval_token_usage=eval_token_usage,
|
|
case_outcomes={
|
|
case_id: CaseOutcomeSummary(
|
|
passed=o.passed, connectivity=o.connectivity, abandoned=o.abandoned
|
|
)
|
|
for case_id, o in case_outcomes.items()
|
|
},
|
|
case_errors=case_errors or [],
|
|
model_configs=model_configs or {},
|
|
)
|