"""Unit tests for resolve_case_verdicts — the single case-verdict read seam.""" from agenteval.evaluation.case_verdict import CaseEvidence, build_case_evidence, resolve_case_verdicts from agenteval.models import CaseOutcomeSummary def _ev(has_turns=True, all_replied=True, result_passes=()): return CaseEvidence(has_turns=has_turns, all_replied=all_replied, result_passes=tuple(result_passes)) # ── authoritative wins ────────────────────────────────────────────────────── def test_authoritative_outcome_used_verbatim_ignoring_evidence(): # Evidence would approximate passed=True (all rules pass), but the authoritative # verdict says failed (e.g. WEIGHTED/ANY logic) — authority must win. verdicts = resolve_case_verdicts( case_outcomes={"c1": CaseOutcomeSummary(passed=False, connectivity=False)}, evidence={"c1": _ev(result_passes=(True, True))}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False) def test_authoritative_connectivity_preserved(): verdicts = resolve_case_verdicts( case_outcomes={"c1": CaseOutcomeSummary(passed=True, connectivity=True)}, evidence={"c1": _ev(result_passes=(False,))}, errored_case_ids=set(), ) assert verdicts["c1"].connectivity is True assert verdicts["c1"].passed is True # ── legacy approximation (case_outcomes missing) ──────────────────────────── def test_legacy_connectivity_case_passes(): # No judged results, has turns, every turn replied, not errored → connectivity pass. verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(has_turns=True, all_replied=True, result_passes=())}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=True, connectivity=True) def test_legacy_all_rules_pass(): verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(result_passes=(True, True))}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=True, connectivity=False) def test_legacy_some_rule_fails(): verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(result_passes=(True, False))}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False) def test_legacy_fault_no_results_not_all_replied(): # No results and a turn missing its reply → fault, not connectivity → fail (ADR-0002). verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(has_turns=True, all_replied=False, result_passes=())}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False) def test_legacy_errored_case_not_connectivity(): # A case with a case-level error is not connectivity even if it replied. verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(has_turns=True, all_replied=True, result_passes=())}, errored_case_ids={"c1"}, ) assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False) def test_legacy_no_turns_no_results_is_fault(): verdicts = resolve_case_verdicts( case_outcomes={}, evidence={"c1": _ev(has_turns=False, all_replied=True, result_passes=())}, errored_case_ids=set(), ) assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False) # ── mixed authoritative + legacy in one call ──────────────────────────────── def test_mixed_authoritative_and_legacy(): verdicts = resolve_case_verdicts( case_outcomes={"auth": CaseOutcomeSummary(passed=True, connectivity=False)}, evidence={ "auth": _ev(result_passes=(False,)), # authority overrides "legacy": _ev(result_passes=(True,)), # approximated }, errored_case_ids=set(), ) assert verdicts["auth"] == CaseOutcomeSummary(passed=True, connectivity=False) assert verdicts["legacy"] == CaseOutcomeSummary(passed=True, connectivity=False) # ── build_case_evidence ────────────────────────────────────────────────────── class _FakeTurn: def __init__(self, case_id: str, has_reply: bool = True): self.case_id = case_id self._reply = {"msgBody": {"content": "ok"}} if has_reply else None def get_reply(self): return self._reply class _FakeResult: def __init__(self, case_id: str, passed: bool): self.case_id = case_id self.passed = passed def test_build_evidence_turns_only_no_results(): """Turns present, no results → has_turns=True, result_passes=().""" turns = [_FakeTurn("c-1"), _FakeTurn("c-1")] results: list = [] evidence = build_case_evidence(turns, results) assert evidence == { "c-1": CaseEvidence(has_turns=True, all_replied=True, result_passes=()), } def test_build_evidence_missing_reply_sets_all_replied_false(): """A turn with no reply → all_replied=False for that case.""" turns = [_FakeTurn("c-1", has_reply=True), _FakeTurn("c-1", has_reply=False)] results: list = [] evidence = build_case_evidence(turns, results) assert evidence["c-1"].all_replied is False assert evidence["c-1"].has_turns is True def test_build_evidence_turns_and_results_aggregate_passes(): """Turns + results → result_passes aggregates all rule passes for the case.""" turns = [_FakeTurn("c-1")] results = [_FakeResult("c-1", passed=True), _FakeResult("c-1", passed=False)] evidence = build_case_evidence(turns, results) assert evidence["c-1"].result_passes == (True, False) assert evidence["c-1"].has_turns is True assert evidence["c-1"].all_replied is True def test_build_evidence_results_without_turns(): """Results exist but no turns → has_turns=False.""" turns: list = [] results = [_FakeResult("c-1", passed=True)] evidence = build_case_evidence(turns, results) assert evidence["c-1"].has_turns is False assert evidence["c-1"].all_replied is True assert evidence["c-1"].result_passes == (True,) def test_build_evidence_multiple_cases(): """Multiple cases are grouped independently.""" turns = [_FakeTurn("c-1"), _FakeTurn("c-2", has_reply=False)] results = [_FakeResult("c-1", passed=True), _FakeResult("c-2", passed=False)] evidence = build_case_evidence(turns, results) assert len(evidence) == 2 assert evidence["c-1"] == CaseEvidence(has_turns=True, all_replied=True, result_passes=(True,)) assert evidence["c-2"] == CaseEvidence(has_turns=True, all_replied=False, result_passes=(False,))