diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index 1f56c77..0fb94e2 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -15,6 +15,7 @@ from agenteval.channels.factory import ChannelFactory from agenteval.config import get_settings from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome from agenteval.evaluation.rules import RuleResult, get_rule +from agenteval.evaluation.run_summary import build_run_summary from agenteval.model_gateway import ModelGateway from agenteval.models import ( Case, @@ -130,9 +131,7 @@ class EvalEngine: try: total_cases = len(self.scenario.cases) - passed_cases = 0 - failed_cases = 0 - case_outcomes: dict[str, dict[str, bool]] = {} + case_outcomes: dict[str, CaseOutcome] = {} for idx, case in enumerate(self.scenario.cases, start=1): self._check_cancel() @@ -151,11 +150,7 @@ class EvalEngine: case, progress_callback, ) - case_outcomes[case.id] = {"passed": outcome.passed, "connectivity": outcome.connectivity} - if outcome.passed: - passed_cases += 1 - else: - failed_cases += 1 + case_outcomes[case.id] = outcome await self._emit( progress_callback, "case_end", @@ -170,34 +165,18 @@ class EvalEngine: ) results = self.run_repo.get_results(run.id) - total_rules = len(results) - passed_rules = sum(1 for r in results if r.passed) - turns = self.run_repo.get_turns(run.id) - latencies = [t.latency_ms for t in turns if t.latency_ms is not None] - avg_latency_ms = round(sum(latencies) / len(latencies), 1) if latencies else None - - summary = { - "total_cases": total_cases, - "passed_cases": passed_cases, - "failed_cases": failed_cases, - "total_rules": total_rules, - "passed_rules": passed_rules, - # 通过率是用例级口径（CONTEXT.md）；规则级数字保留在 passed_rules/total_rules - "pass_rate": round(passed_cases / total_cases, 4) if total_cases else 0.0, - # 平均时延（毫秒），供活动周期报告的时延轴聚合；无回复轮不计入 - "avg_latency_ms": avg_latency_ms, - # 逐用例权威判定（judgement.py 算一次），报告/对比/渲染层只读不重算 - "case_outcomes": case_outcomes, - } - # Surface fatal case-level errors (e.g. dynamic generation failures) - # so the report / DB record shows *why* a run produced no results. - if self._case_errors: - summary["case_errors"] = self._case_errors - if self._resolved_models: - summary["model_configs"] = { - purpose.value: config.snapshot() for purpose, config in self._resolved_models.items() - } + summary = build_run_summary( + case_outcomes=case_outcomes, + latencies=[t.latency_ms for t in turns if t.latency_ms is not None], + rule_passes=[r.passed for r in results], + case_errors=self._case_errors or None, + model_configs=( + {purpose.value: config.snapshot() for purpose, config in self._resolved_models.items()} + if self._resolved_models + else None + ), + ) run.status = RunStatus.COMPLETED run.completed_at = utc_now() run.summary = summary @@ -206,7 +185,7 @@ class EvalEngine: "run_completed", { "status": "completed", - "summary": summary, + "summary": summary.model_dump(), }, ) except CancelledError: diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index a75c4a8..1e3adf6 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -103,8 +103,10 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: passed_cases = summary.passed_cases connectivity_count = sum(1 for c in cases if c["connectivity"]) judged_total = total_cases - connectivity_count - # 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数 - judged_pass_rate = round((passed_cases - connectivity_count) / judged_total, 4) if judged_total > 0 else None + # 判定型通过率由 build_run_summary 入库，读路径只读；老 run 缺字段时按同一口径回退近似 + judged_pass_rate = summary.judged_pass_rate + if judged_pass_rate is None and judged_total > 0: + judged_pass_rate = round((passed_cases - connectivity_count) / judged_total, 4) return { "run_id": run.id, diff --git a/backend/agenteval/evaluation/run_summary.py b/backend/agenteval/evaluation/run_summary.py new file mode 100644 index 0000000..91e0a91 --- /dev/null +++ b/backend/agenteval/evaluation/run_summary.py @@ -0,0 +1,53 @@ +"""Single-run summary aggregation — the pure落点 for one run's口径. + +Parallel to ``metrics.aggregate_runs`` (cross-run) and +``judgement.combine_case_outcome`` (case-level): given the authoritative +per-case outcomes plus raw latency/rule material, compute the run's +summary口径 once. No IO — the engine collects material and calls this; DB +writes and event emits stay in the caller. See CONTEXT.md (通过率) / ADR-0002. +""" + +from typing import Any, Optional, Sequence + +from agenteval.evaluation.judgement import CaseOutcome +from agenteval.models import CaseOutcomeSummary, RunSummary + + +def build_run_summary( + *, + case_outcomes: dict[str, CaseOutcome], + latencies: Sequence[float], + rule_passes: Sequence[bool], + case_errors: Optional[list[dict[str, str]]] = None, + model_configs: Optional[dict[str, Any]] = None, +) -> RunSummary: + """Compute a run's summary口径 from its authoritative case outcomes.""" + total_cases = len(case_outcomes) + passed_cases = sum(1 for o in case_outcomes.values() if o.passed) + connectivity_count = sum(1 for o in case_outcomes.values() if o.connectivity) + + pass_rate = round(passed_cases / total_cases, 4) if total_cases else 0.0 + # 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数 + judged_total = total_cases - connectivity_count + judged_pass_rate = ( + round((passed_cases - connectivity_count) / judged_total, 4) if judged_total > 0 else None + ) + + avg_latency_ms = round(sum(latencies) / len(latencies), 1) if latencies else None + + return RunSummary( + total_cases=total_cases, + passed_cases=passed_cases, + failed_cases=total_cases - passed_cases, + total_rules=len(rule_passes), + passed_rules=sum(1 for p in rule_passes if p), + pass_rate=pass_rate, + judged_pass_rate=judged_pass_rate, + avg_latency_ms=avg_latency_ms, + case_outcomes={ + case_id: CaseOutcomeSummary(passed=o.passed, connectivity=o.connectivity) + for case_id, o in case_outcomes.items() + }, + case_errors=case_errors or [], + model_configs=model_configs or {}, + ) diff --git a/backend/agenteval/models.py b/backend/agenteval/models.py index 5682d53..8c4b35a 100644 --- a/backend/agenteval/models.py +++ b/backend/agenteval/models.py @@ -187,6 +187,8 @@ class RunSummary(BaseModel): passed_rules: int = 0 # 用例级通过率，含执行失败（ADR-0002）；失败/取消的 run 无此值 pass_rate: Optional[float] = None + # 判定型通过率：连通用例从分子分母双双剔除；无判定型用例时为空 + judged_pass_rate: Optional[float] = None avg_latency_ms: Optional[float] = None case_outcomes: dict[str, CaseOutcomeSummary] = Field(default_factory=dict) case_errors: list[dict[str, str]] = Field(default_factory=list) diff --git a/tests/unit/test_build_run_summary.py b/tests/unit/test_build_run_summary.py new file mode 100644 index 0000000..f07cbaf --- /dev/null +++ b/tests/unit/test_build_run_summary.py @@ -0,0 +1,116 @@ +"""Unit tests for the pure single-run summary seam. + +build_run_summary is the single reusable落点 for one run's口径 (pass_rate / +judged_pass_rate / avg_latency / connectivity split), parallel to +metrics.aggregate_runs (cross-run) and judgement.combine_case_outcome +(case-level). No IO — fed constructed material, asserted directly. +""" + +from agenteval.evaluation.judgement import CaseOutcome +from agenteval.evaluation.run_summary import build_run_summary + + +def _outcomes(**kw: CaseOutcome) -> dict[str, CaseOutcome]: + return dict(kw) + + +def test_all_judged_passing(): + summary = build_run_summary( + case_outcomes=_outcomes( + a=CaseOutcome(passed=True, connectivity=False), + b=CaseOutcome(passed=True, connectivity=False), + ), + latencies=[100.0, 300.0], + rule_passes=[True, True, True], + ) + assert summary.total_cases == 2 + assert summary.passed_cases == 2 + assert summary.failed_cases == 0 + assert summary.pass_rate == 1.0 + assert summary.judged_pass_rate == 1.0 + assert summary.total_rules == 3 + assert summary.passed_rules == 3 + assert summary.avg_latency_ms == 200.0 + + +def test_mixed_pass_fail(): + summary = build_run_summary( + case_outcomes=_outcomes( + a=CaseOutcome(passed=True, connectivity=False), + b=CaseOutcome(passed=False, connectivity=False), + c=CaseOutcome(passed=False, connectivity=False), + ), + latencies=[], + rule_passes=[True, False], + ) + assert summary.total_cases == 3 + assert summary.passed_cases == 1 + assert summary.failed_cases == 2 + assert summary.pass_rate == 0.3333 + assert summary.judged_pass_rate == 0.3333 + assert summary.passed_rules == 1 + assert summary.total_rules == 2 + assert summary.avg_latency_ms is None + + +def test_connectivity_excluded_from_judged_rate(): + # 连通用例计入总通过率，但从判定型口径的分子分母双双剔除（report.py 现口径） + summary = build_run_summary( + case_outcomes=_outcomes( + conn=CaseOutcome(passed=True, connectivity=True), + judged_pass=CaseOutcome(passed=True, connectivity=False), + judged_fail=CaseOutcome(passed=False, connectivity=False), + ), + latencies=[50.0], + rule_passes=[True, False], + ) + assert summary.total_cases == 3 + assert summary.passed_cases == 2 # conn + judged_pass + assert summary.pass_rate == 0.6667 + # judged: 分母 = 3 - 1 连通 = 2；分子 = 2 通过 - 1 连通 = 1 + assert summary.judged_pass_rate == 0.5 + + +def test_all_connectivity_yields_no_judged_rate(): + summary = build_run_summary( + case_outcomes=_outcomes( + a=CaseOutcome(passed=True, connectivity=True), + b=CaseOutcome(passed=True, connectivity=True), + ), + latencies=[10.0], + rule_passes=[], + ) + assert summary.pass_rate == 1.0 + assert summary.judged_pass_rate is None # judged_total == 0 + + +def test_empty_run(): + summary = build_run_summary(case_outcomes={}, latencies=[], rule_passes=[]) + assert summary.total_cases == 0 + assert summary.pass_rate == 0.0 + assert summary.judged_pass_rate is None + assert summary.avg_latency_ms is None + + +def test_case_errors_and_model_configs_passed_through(): + errors = [{"case_id": "x", "message": "boom"}] + configs = {"judge": {"model": "gpt-4o"}} + summary = build_run_summary( + case_outcomes=_outcomes(x=CaseOutcome(passed=False, connectivity=False)), + latencies=[], + rule_passes=[], + case_errors=errors, + model_configs=configs, + ) + assert summary.case_errors == errors + assert summary.model_configs == configs + + +def test_case_outcomes_persisted_as_verdict_snapshot(): + summary = build_run_summary( + case_outcomes=_outcomes(a=CaseOutcome(passed=True, connectivity=True)), + latencies=[], + rule_passes=[], + ) + assert summary.case_outcomes["a"].passed is True + assert summary.case_outcomes["a"].connectivity is True