AgentEvalTool/tests/unit/test_case_verdict.py

"""Unit tests for resolve_case_verdicts — the single case-verdict read seam."""

from agenteval.evaluation.case_verdict import CaseEvidence, build_case_evidence, resolve_case_verdicts
from agenteval.models import CaseOutcomeSummary


def _ev(has_turns=True, all_replied=True, result_passes=()):
    return CaseEvidence(has_turns=has_turns, all_replied=all_replied, result_passes=tuple(result_passes))


# ── authoritative wins ──────────────────────────────────────────────────────

def test_authoritative_outcome_used_verbatim_ignoring_evidence():
    # Evidence would approximate passed=True (all rules pass), but the authoritative
    # verdict says failed (e.g. WEIGHTED/ANY logic) — authority must win.
    verdicts = resolve_case_verdicts(
        case_outcomes={"c1": CaseOutcomeSummary(passed=False, connectivity=False)},
        evidence={"c1": _ev(result_passes=(True, True))},
        errored_case_ids=set(),
    )
    assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False)


def test_authoritative_connectivity_preserved():
    verdicts = resolve_case_verdicts(
        case_outcomes={"c1": CaseOutcomeSummary(passed=True, connectivity=True)},
        evidence={"c1": _ev(result_passes=(False,))},
        errored_case_ids=set(),
    )
    assert verdicts["c1"].connectivity is True
    assert verdicts["c1"].passed is True


# ── legacy approximation (case_outcomes missing) ────────────────────────────

def test_legacy_connectivity_case_passes():
    # No judged results, has turns, every turn replied, not errored → connectivity pass.
    verdicts = resolve_case_verdicts(
        case_outcomes={},
        evidence={"c1": _ev(has_turns=True, all_replied=True, result_passes=())},
        errored_case_ids=set(),
    )
    assert verdicts["c1"] == CaseOutcomeSummary(passed=True, connectivity=True)


def test_legacy_all_rules_pass():
    verdicts = resolve_case_verdicts(
        case_outcomes={},
        evidence={"c1": _ev(result_passes=(True, True))},
        errored_case_ids=set(),
    )
    assert verdicts["c1"] == CaseOutcomeSummary(passed=True, connectivity=False)


def test_legacy_some_rule_fails():
    verdicts = resolve_case_verdicts(
        case_outcomes={},
        evidence={"c1": _ev(result_passes=(True, False))},
        errored_case_ids=set(),
    )
    assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False)


def test_legacy_fault_no_results_not_all_replied():
    # No results and a turn missing its reply → fault, not connectivity → fail (ADR-0002).
    verdicts = resolve_case_verdicts(
        case_outcomes={},
        evidence={"c1": _ev(has_turns=True, all_replied=False, result_passes=())},
        errored_case_ids=set(),
    )
    assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False)


def test_legacy_errored_case_not_connectivity():
    # A case with a case-level error is not connectivity even if it replied.
    verdicts = resolve_case_verdicts(
        case_outcomes={},
        evidence={"c1": _ev(has_turns=True, all_replied=True, result_passes=())},
        errored_case_ids={"c1"},
    )
    assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False)


def test_legacy_no_turns_no_results_is_fault():
    verdicts = resolve_case_verdicts(
        case_outcomes={},
        evidence={"c1": _ev(has_turns=False, all_replied=True, result_passes=())},
        errored_case_ids=set(),
    )
    assert verdicts["c1"] == CaseOutcomeSummary(passed=False, connectivity=False)


# ── mixed authoritative + legacy in one call ────────────────────────────────

def test_mixed_authoritative_and_legacy():
    verdicts = resolve_case_verdicts(
        case_outcomes={"auth": CaseOutcomeSummary(passed=True, connectivity=False)},
        evidence={
            "auth": _ev(result_passes=(False,)),          # authority overrides
            "legacy": _ev(result_passes=(True,)),          # approximated
        },
        errored_case_ids=set(),
    )
    assert verdicts["auth"] == CaseOutcomeSummary(passed=True, connectivity=False)
    assert verdicts["legacy"] == CaseOutcomeSummary(passed=True, connectivity=False)


# ── build_case_evidence ──────────────────────────────────────────────────────


class _FakeTurn:
    def __init__(self, case_id: str, has_reply: bool = True):
        self.case_id = case_id
        self._reply = {"msgBody": {"content": "ok"}} if has_reply else None

    def get_reply(self):
        return self._reply


class _FakeResult:
    def __init__(self, case_id: str, passed: bool):
        self.case_id = case_id
        self.passed = passed


def test_build_evidence_turns_only_no_results():
    """Turns present, no results → has_turns=True, result_passes=()."""
    turns = [_FakeTurn("c-1"), _FakeTurn("c-1")]
    results: list = []
    evidence = build_case_evidence(turns, results)
    assert evidence == {
        "c-1": CaseEvidence(has_turns=True, all_replied=True, result_passes=()),
    }


def test_build_evidence_missing_reply_sets_all_replied_false():
    """A turn with no reply → all_replied=False for that case."""
    turns = [_FakeTurn("c-1", has_reply=True), _FakeTurn("c-1", has_reply=False)]
    results: list = []
    evidence = build_case_evidence(turns, results)
    assert evidence["c-1"].all_replied is False
    assert evidence["c-1"].has_turns is True


def test_build_evidence_turns_and_results_aggregate_passes():
    """Turns + results → result_passes aggregates all rule passes for the case."""
    turns = [_FakeTurn("c-1")]
    results = [_FakeResult("c-1", passed=True), _FakeResult("c-1", passed=False)]
    evidence = build_case_evidence(turns, results)
    assert evidence["c-1"].result_passes == (True, False)
    assert evidence["c-1"].has_turns is True
    assert evidence["c-1"].all_replied is True


def test_build_evidence_results_without_turns():
    """Results exist but no turns → has_turns=False."""
    turns: list = []
    results = [_FakeResult("c-1", passed=True)]
    evidence = build_case_evidence(turns, results)
    assert evidence["c-1"].has_turns is False
    assert evidence["c-1"].all_replied is True
    assert evidence["c-1"].result_passes == (True,)


def test_build_evidence_multiple_cases():
    """Multiple cases are grouped independently."""
    turns = [_FakeTurn("c-1"), _FakeTurn("c-2", has_reply=False)]
    results = [_FakeResult("c-1", passed=True), _FakeResult("c-2", passed=False)]
    evidence = build_case_evidence(turns, results)
    assert len(evidence) == 2
    assert evidence["c-1"] == CaseEvidence(has_turns=True, all_replied=True, result_passes=(True,))
    assert evidence["c-2"] == CaseEvidence(has_turns=True, all_replied=False, result_passes=(False,))