782916a283
Give EvalRun.summary a typed RunSummary value (unified RunError, lenient legacy parsing) so readers stop reaching into a schemaless dict, and route every cross-run rollup — dashboard, scenario ranking, trend, campaign report — through one aggregate_runs seam. Fixes the divergence where stats averaged pass_rate over completed-only runs while the campaign report counted faults as 0.0. Cross-run rule (ADR-0004): genuine faults count 0.0, user-cancelled runs are excluded from both denominators.
130 lines
4.7 KiB
Python
130 lines
4.7 KiB
Python
"""Unit tests for the RunSummary value type and the single cross-run
|
|
aggregation seam (ADR-0004: faults count 0.0, user-cancelled excluded)."""
|
|
|
|
from agenteval.evaluation.metrics import aggregate_runs
|
|
from agenteval.models import EvalRun, RunStatus, RunSummary
|
|
|
|
# ── RunSummary parsing (legacy dict rows must keep parsing) ────────────────
|
|
|
|
|
|
def test_parses_full_engine_summary():
|
|
s = RunSummary.model_validate({
|
|
"total_cases": 3,
|
|
"passed_cases": 2,
|
|
"failed_cases": 1,
|
|
"total_rules": 5,
|
|
"passed_rules": 4,
|
|
"pass_rate": 0.6667,
|
|
"avg_latency_ms": 123.4,
|
|
"case_outcomes": {"c1": {"passed": True, "connectivity": False}},
|
|
"case_errors": [{"case_id": "c2", "stage": "generate_messages", "error": "boom"}],
|
|
"model_configs": {"judge": {"model": "m"}},
|
|
})
|
|
assert s.pass_rate == 0.6667
|
|
assert s.avg_latency_ms == 123.4
|
|
assert s.case_outcomes["c1"].passed is True
|
|
assert s.case_outcomes["c1"].connectivity is False
|
|
assert s.error is None
|
|
|
|
|
|
def test_parses_legacy_string_error():
|
|
s = RunSummary.model_validate({"error": "channel exploded"})
|
|
assert s.error is not None
|
|
assert s.error.code == "error"
|
|
assert s.error.message == "channel exploded"
|
|
assert s.is_cancelled is False
|
|
|
|
|
|
def test_parses_object_error_and_detects_cancel():
|
|
s = RunSummary.model_validate({"error": {"code": "cancelled_by_user", "message": "评测已手动停止"}})
|
|
assert s.error is not None
|
|
assert s.error.code == "cancelled_by_user"
|
|
assert s.is_cancelled is True
|
|
|
|
|
|
def test_missing_keys_default_and_unknown_keys_tolerated():
|
|
s = RunSummary.model_validate({"pass_rate": 0.5, "some_future_key": 1})
|
|
assert s.pass_rate == 0.5
|
|
assert s.total_cases == 0
|
|
assert s.case_outcomes == {}
|
|
assert s.error is None
|
|
|
|
|
|
def test_eval_run_summary_field_is_typed():
|
|
run = EvalRun(target_id="t", scenario_id="s", summary={"pass_rate": 1.0})
|
|
assert isinstance(run.summary, RunSummary)
|
|
assert run.summary.pass_rate == 1.0
|
|
|
|
|
|
# ── aggregate_runs (ADR-0004) ──────────────────────────────────────────────
|
|
|
|
|
|
def _run(status, *, pass_rate=None, latency=None, error=None) -> EvalRun:
|
|
summary = None
|
|
if pass_rate is not None or latency is not None or error is not None:
|
|
summary = {}
|
|
if pass_rate is not None:
|
|
summary["pass_rate"] = pass_rate
|
|
if latency is not None:
|
|
summary["avg_latency_ms"] = latency
|
|
if error is not None:
|
|
summary["error"] = error
|
|
return EvalRun(target_id="t", scenario_id="s", status=status, summary=summary)
|
|
|
|
|
|
def test_empty_runs_aggregate_to_none():
|
|
agg = aggregate_runs([])
|
|
assert agg == {"run_count": 0, "pass_rate": None, "availability": None, "avg_latency_ms": None}
|
|
|
|
|
|
def test_completed_runs_average_pass_rate_and_latency():
|
|
agg = aggregate_runs([
|
|
_run(RunStatus.COMPLETED, pass_rate=1.0, latency=100),
|
|
_run(RunStatus.COMPLETED, pass_rate=0.5, latency=200),
|
|
])
|
|
assert agg["run_count"] == 2
|
|
assert agg["pass_rate"] == 0.75
|
|
assert agg["availability"] == 1.0
|
|
assert agg["avg_latency_ms"] == 150.0
|
|
|
|
|
|
def test_faulted_run_counts_zero_in_both_denominators():
|
|
# ADR-0004: a genuine execution fault drags pass_rate AND availability down.
|
|
agg = aggregate_runs([
|
|
_run(RunStatus.COMPLETED, pass_rate=1.0),
|
|
_run(RunStatus.FAILED, error="channel exploded"),
|
|
])
|
|
assert agg["pass_rate"] == 0.5
|
|
assert agg["availability"] == 0.5
|
|
|
|
|
|
def test_cancelled_run_excluded_from_denominators():
|
|
# ADR-0004: user cancellation is neither a quality nor availability signal.
|
|
agg = aggregate_runs([
|
|
_run(RunStatus.COMPLETED, pass_rate=1.0),
|
|
_run(RunStatus.FAILED, error={"code": "cancelled_by_user", "message": "stop"}),
|
|
])
|
|
assert agg["run_count"] == 2 # what happened stays visible
|
|
assert agg["pass_rate"] == 1.0 # cancelled run out of the denominator
|
|
assert agg["availability"] == 1.0
|
|
|
|
|
|
def test_all_cancelled_aggregates_to_none():
|
|
agg = aggregate_runs([
|
|
_run(RunStatus.FAILED, error={"code": "cancelled_by_user", "message": "stop"}),
|
|
])
|
|
assert agg["run_count"] == 1
|
|
assert agg["pass_rate"] is None
|
|
assert agg["availability"] is None
|
|
assert agg["avg_latency_ms"] is None
|
|
|
|
|
|
def test_in_flight_runs_count_zero_not_cancelled():
|
|
# pending/running (no summary) are not cancelled — they count 0.0 for now.
|
|
agg = aggregate_runs([
|
|
_run(RunStatus.RUNNING),
|
|
_run(RunStatus.PENDING),
|
|
])
|
|
assert agg["pass_rate"] == 0.0
|
|
assert agg["availability"] == 0.0
|