"""Unit tests for resolve_case_verdicts — the single case-verdict read seam.""" from agenteval.evaluation.case_verdict import CaseEvidence, resolve_case_verdicts from agenteval.models import CaseOutcomeSummary def _ev(has_turns=True, all_replied=True, result_passes=()): return CaseEvidence(has_turns=has_turns, all_replied=all_replied, result_passes=tuple(result_passes)) # ── authoritative wins ────────────────────────────────────────────────────── def test_authoritative_outcome_used_verbatim_ignoring_evidence(): # Evidence would approximate passed=True (all rules pass), but the authoritative # verdict says failed (e.g. WEIGHTED/ANY logic) — authority must win. verdicts = resolve_case_verdicts( case_outcomes={"c1": CaseOutcomeSummary(passed=False, connectivity=False)}, evidence={"c1": _ev(result_passes=(True, True))}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False) def test_authoritative_connectivity_preserved(): verdicts = resolve_case_verdicts( case_outcomes={"c1": CaseOutcomeSummary(passed=True, connectivity=True)}, evidence={"c1": _ev(result_passes=(False,))}, errored_case_ids=set(), ) assert verdicts["c1"].connectivity is True assert verdicts["c1"].passed is True # ── legacy approximation (case_outcomes missing) ──────────────────────────── def test_legacy_connectivity_case_passes(): # No judged results, has turns, every turn replied, not errored → connectivity pass. verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(has_turns=True, all_replied=True, result_passes=())}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=True, connectivity=True) def test_legacy_all_rules_pass(): verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(result_passes=(True, True))}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=True, connectivity=False) def test_legacy_some_rule_fails(): verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(result_passes=(True, False))}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False) def test_legacy_fault_no_results_not_all_replied(): # No results and a turn missing its reply → fault, not connectivity → fail (ADR-0002). verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(has_turns=True, all_replied=False, result_passes=())}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False) def test_legacy_errored_case_not_connectivity(): # A case with a case-level error is not connectivity even if it replied. verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(has_turns=True, all_replied=True, result_passes=())}, errored_case_ids={"c1"}, ) assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False) def test_legacy_no_turns_no_results_is_fault(): verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(has_turns=False, all_replied=True, result_passes=())}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False) # ── mixed authoritative + legacy in one call ──────────────────────────────── def test_mixed_authoritative_and_legacy(): verdicts = resolve_case_verdicts( case_outcomes={"auth": CaseOutcomeSummary(passed=True, connectivity=False)}, evidence={ "auth": _ev(result_passes=(False,)), # authority overrides "legacy": _ev(result_passes=(True,)), # approximated }, errored_case_ids=set(), ) assert verdicts["auth"] == CaseOutcomeSummary(passed=True, connectivity=False) assert verdicts["legacy"] == CaseOutcomeSummary(passed=True, connectivity=False)