9c01afa79b
Single-run summary口径 (pass_rate / judged_pass_rate / avg_latency / connectivity split) was inlined in run(), reachable only by driving a whole async run, and report.py recomputed judged_pass_rate independently. Extract build_run_summary — a pure function parallel to aggregate_runs (cross-run) and combine_case_outcome (case-level). run() now collects material and delegates; judged_pass_rate is stored in RunSummary so the report reads it instead of recomputing.
54 lines
2.1 KiB
Python
54 lines
2.1 KiB
Python
"""Single-run summary aggregation — the pure落点 for one run's口径.
|
|
|
|
Parallel to ``metrics.aggregate_runs`` (cross-run) and
|
|
``judgement.combine_case_outcome`` (case-level): given the authoritative
|
|
per-case outcomes plus raw latency/rule material, compute the run's
|
|
summary口径 once. No IO — the engine collects material and calls this; DB
|
|
writes and event emits stay in the caller. See CONTEXT.md (通过率) / ADR-0002.
|
|
"""
|
|
|
|
from typing import Any, Optional, Sequence
|
|
|
|
from agenteval.evaluation.judgement import CaseOutcome
|
|
from agenteval.models import CaseOutcomeSummary, RunSummary
|
|
|
|
|
|
def build_run_summary(
|
|
*,
|
|
case_outcomes: dict[str, CaseOutcome],
|
|
latencies: Sequence[float],
|
|
rule_passes: Sequence[bool],
|
|
case_errors: Optional[list[dict[str, str]]] = None,
|
|
model_configs: Optional[dict[str, Any]] = None,
|
|
) -> RunSummary:
|
|
"""Compute a run's summary口径 from its authoritative case outcomes."""
|
|
total_cases = len(case_outcomes)
|
|
passed_cases = sum(1 for o in case_outcomes.values() if o.passed)
|
|
connectivity_count = sum(1 for o in case_outcomes.values() if o.connectivity)
|
|
|
|
pass_rate = round(passed_cases / total_cases, 4) if total_cases else 0.0
|
|
# 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数
|
|
judged_total = total_cases - connectivity_count
|
|
judged_pass_rate = (
|
|
round((passed_cases - connectivity_count) / judged_total, 4) if judged_total > 0 else None
|
|
)
|
|
|
|
avg_latency_ms = round(sum(latencies) / len(latencies), 1) if latencies else None
|
|
|
|
return RunSummary(
|
|
total_cases=total_cases,
|
|
passed_cases=passed_cases,
|
|
failed_cases=total_cases - passed_cases,
|
|
total_rules=len(rule_passes),
|
|
passed_rules=sum(1 for p in rule_passes if p),
|
|
pass_rate=pass_rate,
|
|
judged_pass_rate=judged_pass_rate,
|
|
avg_latency_ms=avg_latency_ms,
|
|
case_outcomes={
|
|
case_id: CaseOutcomeSummary(passed=o.passed, connectivity=o.connectivity)
|
|
for case_id, o in case_outcomes.items()
|
|
},
|
|
case_errors=case_errors or [],
|
|
model_configs=model_configs or {},
|
|
)
|