2fddce8c92
用例判定证据构建收敛到 case_verdict.py 的 build_case_evidence 纯函数， report.py 和 runs.py 各删 ~15 行重复逻辑，换一行调用。locality 回归： 证据构建改一处，全局生效。 - 新增 build_case_evidence(turns, results) -> dict[str, CaseEvidence] - report.py:76-83 证据构建替换为一行调用 - runs.py:175-201 证据构建替换为一行调用 - 5 个新测试覆盖 build_case_evidence（纯函数，无 DB 依赖）
97 lines
3.5 KiB
Python
97 lines
3.5 KiB
Python
"""Case-verdict read seam — the single place the read path derives pass/connectivity.
|
|
|
|
The engine is the authority: it runs ``judgement.combine_case_outcome`` once and
|
|
writes each case's verdict into ``summary.case_outcomes``. Every read surface
|
|
(report generation, the run-logs endpoint) must present *that* verdict, never
|
|
recompute it — otherwise WEIGHTED/ANY logic and connectivity cases diverge from
|
|
what was judged.
|
|
|
|
This module is that single seam. It reads the authoritative ``case_outcomes``
|
|
when present, and only for older runs that predate it falls back to a documented
|
|
approximation from persisted turns/results. Pure — no I/O; callers build the
|
|
per-case ``CaseEvidence`` from whatever they already have in hand.
|
|
"""
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from agenteval.models import CaseOutcomeSummary
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CaseEvidence:
|
|
"""What the legacy approximation needs about one case's persisted record.
|
|
|
|
``result_passes`` is the per-rule pass flags (empty means no judged rule
|
|
result exists for the case — the connectivity-vs-fault fork).
|
|
"""
|
|
|
|
has_turns: bool
|
|
all_replied: bool
|
|
result_passes: tuple[bool, ...] = ()
|
|
|
|
|
|
def build_case_evidence(turns: list, results: list) -> dict[str, CaseEvidence]:
|
|
"""Group turns + results by case_id into CaseEvidence per case.
|
|
|
|
Pure — no I/O. Callers supply whatever turn/result objects expose
|
|
``.case_id`` and (for turns) ``.get_reply()`` / (for results) ``.passed``.
|
|
"""
|
|
evidence: dict[str, dict] = {}
|
|
for t in turns:
|
|
ev = evidence.get(t.case_id)
|
|
if ev is None:
|
|
ev = {"has_turns": True, "all_replied": True, "passes": []}
|
|
evidence[t.case_id] = ev
|
|
else:
|
|
ev["has_turns"] = True
|
|
if t.get_reply() is None:
|
|
ev["all_replied"] = False
|
|
for r in results:
|
|
ev = evidence.setdefault(r.case_id, {"has_turns": False, "all_replied": True, "passes": []})
|
|
ev["passes"].append(r.passed)
|
|
return {
|
|
cid: CaseEvidence(
|
|
has_turns=ev["has_turns"],
|
|
all_replied=ev["all_replied"],
|
|
result_passes=tuple(ev["passes"]),
|
|
)
|
|
for cid, ev in evidence.items()
|
|
}
|
|
|
|
|
|
def resolve_case_verdicts(
|
|
*,
|
|
case_outcomes: dict[str, CaseOutcomeSummary],
|
|
evidence: dict[str, CaseEvidence],
|
|
errored_case_ids: set[str],
|
|
) -> dict[str, CaseOutcomeSummary]:
|
|
"""Resolve every case in ``evidence`` to its authoritative-or-approximated verdict.
|
|
|
|
Authoritative ``case_outcomes`` win verbatim. For a case missing from it (an
|
|
older run), approximate per CONTEXT.md / ADR-0002: a case with no judged
|
|
results but turns that all replied and no case-level error is a *connectivity*
|
|
case (counts as passed); a case with results passes iff every rule passed;
|
|
anything else (a fault) fails.
|
|
"""
|
|
verdicts: dict[str, CaseOutcomeSummary] = {}
|
|
for case_id, ev in evidence.items():
|
|
authoritative = case_outcomes.get(case_id)
|
|
if authoritative is not None:
|
|
verdicts[case_id] = authoritative
|
|
continue
|
|
|
|
connectivity = (
|
|
not ev.result_passes
|
|
and ev.has_turns
|
|
and ev.all_replied
|
|
and case_id not in errored_case_ids
|
|
)
|
|
if connectivity:
|
|
passed = True
|
|
elif not ev.result_passes:
|
|
passed = False
|
|
else:
|
|
passed = all(ev.result_passes)
|
|
verdicts[case_id] = CaseOutcomeSummary(passed=passed, connectivity=connectivity)
|
|
return verdicts
|