9c01afa79b
Single-run summary口径 (pass_rate / judged_pass_rate / avg_latency / connectivity split) was inlined in run(), reachable only by driving a whole async run, and report.py recomputed judged_pass_rate independently. Extract build_run_summary — a pure function parallel to aggregate_runs (cross-run) and combine_case_outcome (case-level). run() now collects material and delegates; judged_pass_rate is stored in RunSummary so the report reads it instead of recomputing.
117 lines
4.0 KiB
Python
117 lines
4.0 KiB
Python
"""Unit tests for the pure single-run summary seam.
|
||
|
||
build_run_summary is the single reusable落点 for one run's口径 (pass_rate /
|
||
judged_pass_rate / avg_latency / connectivity split), parallel to
|
||
metrics.aggregate_runs (cross-run) and judgement.combine_case_outcome
|
||
(case-level). No IO — fed constructed material, asserted directly.
|
||
"""
|
||
|
||
from agenteval.evaluation.judgement import CaseOutcome
|
||
from agenteval.evaluation.run_summary import build_run_summary
|
||
|
||
|
||
def _outcomes(**kw: CaseOutcome) -> dict[str, CaseOutcome]:
|
||
return dict(kw)
|
||
|
||
|
||
def test_all_judged_passing():
|
||
summary = build_run_summary(
|
||
case_outcomes=_outcomes(
|
||
a=CaseOutcome(passed=True, connectivity=False),
|
||
b=CaseOutcome(passed=True, connectivity=False),
|
||
),
|
||
latencies=[100.0, 300.0],
|
||
rule_passes=[True, True, True],
|
||
)
|
||
assert summary.total_cases == 2
|
||
assert summary.passed_cases == 2
|
||
assert summary.failed_cases == 0
|
||
assert summary.pass_rate == 1.0
|
||
assert summary.judged_pass_rate == 1.0
|
||
assert summary.total_rules == 3
|
||
assert summary.passed_rules == 3
|
||
assert summary.avg_latency_ms == 200.0
|
||
|
||
|
||
def test_mixed_pass_fail():
|
||
summary = build_run_summary(
|
||
case_outcomes=_outcomes(
|
||
a=CaseOutcome(passed=True, connectivity=False),
|
||
b=CaseOutcome(passed=False, connectivity=False),
|
||
c=CaseOutcome(passed=False, connectivity=False),
|
||
),
|
||
latencies=[],
|
||
rule_passes=[True, False],
|
||
)
|
||
assert summary.total_cases == 3
|
||
assert summary.passed_cases == 1
|
||
assert summary.failed_cases == 2
|
||
assert summary.pass_rate == 0.3333
|
||
assert summary.judged_pass_rate == 0.3333
|
||
assert summary.passed_rules == 1
|
||
assert summary.total_rules == 2
|
||
assert summary.avg_latency_ms is None
|
||
|
||
|
||
def test_connectivity_excluded_from_judged_rate():
|
||
# 连通用例计入总通过率，但从判定型口径的分子分母双双剔除（report.py 现口径）
|
||
summary = build_run_summary(
|
||
case_outcomes=_outcomes(
|
||
conn=CaseOutcome(passed=True, connectivity=True),
|
||
judged_pass=CaseOutcome(passed=True, connectivity=False),
|
||
judged_fail=CaseOutcome(passed=False, connectivity=False),
|
||
),
|
||
latencies=[50.0],
|
||
rule_passes=[True, False],
|
||
)
|
||
assert summary.total_cases == 3
|
||
assert summary.passed_cases == 2 # conn + judged_pass
|
||
assert summary.pass_rate == 0.6667
|
||
# judged: 分母 = 3 - 1 连通 = 2；分子 = 2 通过 - 1 连通 = 1
|
||
assert summary.judged_pass_rate == 0.5
|
||
|
||
|
||
def test_all_connectivity_yields_no_judged_rate():
|
||
summary = build_run_summary(
|
||
case_outcomes=_outcomes(
|
||
a=CaseOutcome(passed=True, connectivity=True),
|
||
b=CaseOutcome(passed=True, connectivity=True),
|
||
),
|
||
latencies=[10.0],
|
||
rule_passes=[],
|
||
)
|
||
assert summary.pass_rate == 1.0
|
||
assert summary.judged_pass_rate is None # judged_total == 0
|
||
|
||
|
||
def test_empty_run():
|
||
summary = build_run_summary(case_outcomes={}, latencies=[], rule_passes=[])
|
||
assert summary.total_cases == 0
|
||
assert summary.pass_rate == 0.0
|
||
assert summary.judged_pass_rate is None
|
||
assert summary.avg_latency_ms is None
|
||
|
||
|
||
def test_case_errors_and_model_configs_passed_through():
|
||
errors = [{"case_id": "x", "message": "boom"}]
|
||
configs = {"judge": {"model": "gpt-4o"}}
|
||
summary = build_run_summary(
|
||
case_outcomes=_outcomes(x=CaseOutcome(passed=False, connectivity=False)),
|
||
latencies=[],
|
||
rule_passes=[],
|
||
case_errors=errors,
|
||
model_configs=configs,
|
||
)
|
||
assert summary.case_errors == errors
|
||
assert summary.model_configs == configs
|
||
|
||
|
||
def test_case_outcomes_persisted_as_verdict_snapshot():
|
||
summary = build_run_summary(
|
||
case_outcomes=_outcomes(a=CaseOutcome(passed=True, connectivity=True)),
|
||
latencies=[],
|
||
rule_passes=[],
|
||
)
|
||
assert summary.case_outcomes["a"].passed is True
|
||
assert summary.case_outcomes["a"].connectivity is True
|