983a58d013
Read paths recomputed per-case pass/connectivity independently — report generation, the logs endpoint, and the frontend each derived it, and the frontend's every(passed) recompute ignored the engine's authoritative verdict. Extract resolve_case_verdicts: a single pure seam that prefers stored case_outcomes verbatim and approximates only for legacy runs. The logs endpoint now surfaces case_verdicts so the frontend reads instead of recomputing.
68 lines
2.5 KiB
Python
68 lines
2.5 KiB
Python
"""Case-verdict read seam — the single place the read path derives pass/connectivity.
|
|
|
|
The engine is the authority: it runs ``judgement.combine_case_outcome`` once and
|
|
writes each case's verdict into ``summary.case_outcomes``. Every read surface
|
|
(report generation, the run-logs endpoint) must present *that* verdict, never
|
|
recompute it — otherwise WEIGHTED/ANY logic and connectivity cases diverge from
|
|
what was judged.
|
|
|
|
This module is that single seam. It reads the authoritative ``case_outcomes``
|
|
when present, and only for older runs that predate it falls back to a documented
|
|
approximation from persisted turns/results. Pure — no I/O; callers build the
|
|
per-case ``CaseEvidence`` from whatever they already have in hand.
|
|
"""
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from agenteval.models import CaseOutcomeSummary
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CaseEvidence:
|
|
"""What the legacy approximation needs about one case's persisted record.
|
|
|
|
``result_passes`` is the per-rule pass flags (empty means no judged rule
|
|
result exists for the case — the connectivity-vs-fault fork).
|
|
"""
|
|
|
|
has_turns: bool
|
|
all_replied: bool
|
|
result_passes: tuple[bool, ...] = ()
|
|
|
|
|
|
def resolve_case_verdicts(
|
|
*,
|
|
case_outcomes: dict[str, CaseOutcomeSummary],
|
|
evidence: dict[str, CaseEvidence],
|
|
errored_case_ids: set[str],
|
|
) -> dict[str, CaseOutcomeSummary]:
|
|
"""Resolve every case in ``evidence`` to its authoritative-or-approximated verdict.
|
|
|
|
Authoritative ``case_outcomes`` win verbatim. For a case missing from it (an
|
|
older run), approximate per CONTEXT.md / ADR-0002: a case with no judged
|
|
results but turns that all replied and no case-level error is a *connectivity*
|
|
case (counts as passed); a case with results passes iff every rule passed;
|
|
anything else (a fault) fails.
|
|
"""
|
|
verdicts: dict[str, CaseOutcomeSummary] = {}
|
|
for case_id, ev in evidence.items():
|
|
authoritative = case_outcomes.get(case_id)
|
|
if authoritative is not None:
|
|
verdicts[case_id] = authoritative
|
|
continue
|
|
|
|
connectivity = (
|
|
not ev.result_passes
|
|
and ev.has_turns
|
|
and ev.all_replied
|
|
and case_id not in errored_case_ids
|
|
)
|
|
if connectivity:
|
|
passed = True
|
|
elif not ev.result_passes:
|
|
passed = False
|
|
else:
|
|
passed = all(ev.result_passes)
|
|
verdicts[case_id] = CaseOutcomeSummary(passed=passed, connectivity=connectivity)
|
|
return verdicts
|