983a58d013
Read paths recomputed per-case pass/connectivity independently — report generation, the logs endpoint, and the frontend each derived it, and the frontend's every(passed) recompute ignored the engine's authoritative verdict. Extract resolve_case_verdicts: a single pure seam that prefers stored case_outcomes verbatim and approximates only for legacy runs. The logs endpoint now surfaces case_verdicts so the frontend reads instead of recomputing.
106 lines
4.3 KiB
Python
106 lines
4.3 KiB
Python
"""Unit tests for resolve_case_verdicts — the single case-verdict read seam."""
|
|
|
|
from agenteval.evaluation.case_verdict import CaseEvidence, resolve_case_verdicts
|
|
from agenteval.models import CaseOutcomeSummary
|
|
|
|
|
|
def _ev(has_turns=True, all_replied=True, result_passes=()):
|
|
return CaseEvidence(has_turns=has_turns, all_replied=all_replied, result_passes=tuple(result_passes))
|
|
|
|
|
|
# ── authoritative wins ──────────────────────────────────────────────────────
|
|
|
|
def test_authoritative_outcome_used_verbatim_ignoring_evidence():
|
|
# Evidence would approximate passed=True (all rules pass), but the authoritative
|
|
# verdict says failed (e.g. WEIGHTED/ANY logic) — authority must win.
|
|
verdicts = resolve_case_verdicts(
|
|
case_outcomes={"c1": CaseOutcomeSummary(passed=False, connectivity=False)},
|
|
evidence={"c1": _ev(result_passes=(True, True))},
|
|
errored_case_ids=set(),
|
|
)
|
|
assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False)
|
|
|
|
|
|
def test_authoritative_connectivity_preserved():
|
|
verdicts = resolve_case_verdicts(
|
|
case_outcomes={"c1": CaseOutcomeSummary(passed=True, connectivity=True)},
|
|
evidence={"c1": _ev(result_passes=(False,))},
|
|
errored_case_ids=set(),
|
|
)
|
|
assert verdicts["c1"].connectivity is True
|
|
assert verdicts["c1"].passed is True
|
|
|
|
|
|
# ── legacy approximation (case_outcomes missing) ────────────────────────────
|
|
|
|
def test_legacy_connectivity_case_passes():
|
|
# No judged results, has turns, every turn replied, not errored → connectivity pass.
|
|
verdicts = resolve_case_verdicts(
|
|
case_outcomes={},
|
|
evidence={"c1": _ev(has_turns=True, all_replied=True, result_passes=())},
|
|
errored_case_ids=set(),
|
|
)
|
|
assert verdicts["c1"] == CaseOutcomeSummary(passed=True, connectivity=True)
|
|
|
|
|
|
def test_legacy_all_rules_pass():
|
|
verdicts = resolve_case_verdicts(
|
|
case_outcomes={},
|
|
evidence={"c1": _ev(result_passes=(True, True))},
|
|
errored_case_ids=set(),
|
|
)
|
|
assert verdicts["c1"] == CaseOutcomeSummary(passed=True, connectivity=False)
|
|
|
|
|
|
def test_legacy_some_rule_fails():
|
|
verdicts = resolve_case_verdicts(
|
|
case_outcomes={},
|
|
evidence={"c1": _ev(result_passes=(True, False))},
|
|
errored_case_ids=set(),
|
|
)
|
|
assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False)
|
|
|
|
|
|
def test_legacy_fault_no_results_not_all_replied():
|
|
# No results and a turn missing its reply → fault, not connectivity → fail (ADR-0002).
|
|
verdicts = resolve_case_verdicts(
|
|
case_outcomes={},
|
|
evidence={"c1": _ev(has_turns=True, all_replied=False, result_passes=())},
|
|
errored_case_ids=set(),
|
|
)
|
|
assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False)
|
|
|
|
|
|
def test_legacy_errored_case_not_connectivity():
|
|
# A case with a case-level error is not connectivity even if it replied.
|
|
verdicts = resolve_case_verdicts(
|
|
case_outcomes={},
|
|
evidence={"c1": _ev(has_turns=True, all_replied=True, result_passes=())},
|
|
errored_case_ids={"c1"},
|
|
)
|
|
assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False)
|
|
|
|
|
|
def test_legacy_no_turns_no_results_is_fault():
|
|
verdicts = resolve_case_verdicts(
|
|
case_outcomes={},
|
|
evidence={"c1": _ev(has_turns=False, all_replied=True, result_passes=())},
|
|
errored_case_ids=set(),
|
|
)
|
|
assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False)
|
|
|
|
|
|
# ── mixed authoritative + legacy in one call ────────────────────────────────
|
|
|
|
def test_mixed_authoritative_and_legacy():
|
|
verdicts = resolve_case_verdicts(
|
|
case_outcomes={"auth": CaseOutcomeSummary(passed=True, connectivity=False)},
|
|
evidence={
|
|
"auth": _ev(result_passes=(False,)), # authority overrides
|
|
"legacy": _ev(result_passes=(True,)), # approximated
|
|
},
|
|
errored_case_ids=set(),
|
|
)
|
|
assert verdicts["auth"] == CaseOutcomeSummary(passed=True, connectivity=False)
|
|
assert verdicts["legacy"] == CaseOutcomeSummary(passed=True, connectivity=False)
|