5db0ede4f4
Some checks failed
CI / test (push) Failing after 50s
「用例是否通过」此前散落 8 处且互相矛盾：engine 权威判定焊死在持久化里 不可单测；report 聚合/compare/markdown 各自从规则结果反推，规则还不一致 （markdown 用 all([]) 把故障用例误渲染成 ✅）。 - 新增纯函数 evaluation/judgement.combine_case_outcome（RuleOutcome/ CaseOutcome），判定组合脱离通道与 DB 可单测（判定矩阵 14 例） - engine 调用它一次，逐用例权威结果写入 summary.case_outcomes（JSON， 零迁移）；report/compare/markdown 只读权威值，老 run fallback 反推 - 故障用例判 False（ADR-0002）：修正 markdown 的 ✅ bug 与 compare 的 None；顺带修 engine 连通用例无回复也算通过的 bug - pass_rate 口径改为用例级（CONTEXT.md 词条），规则级保留在 passed_rules/total_rules；CLI 对比标签同步更正 - 修 RunRepository.update 漏拷 scenario_version/triggered_by 的字段漂移
65 lines
2.1 KiB
Python
65 lines
2.1 KiB
Python
"""Case judgement — the single authority for "did this case pass".
|
||
|
||
判定语义（CONTEXT.md / spec v0.5 / ADR-0002）：
|
||
- 连通用例（无显式规则、无期望）：每轮收到回复即通过；缺回复=故障=不通过。
|
||
- 显式规则按 rule_logic（ALL / ANY / WEIGHTED）组合；无显式规则时空真。
|
||
- 期望派生的隐式规则是 rule_logic 之外的硬约束，任一失败则用例失败。
|
||
|
||
引擎执行时调用一次，结果写入 run.summary["case_outcomes"]，
|
||
报告 / 对比 / 渲染层只消费该权威值，不得各自重算。
|
||
"""
|
||
|
||
from dataclasses import dataclass
|
||
from typing import Optional, Sequence
|
||
|
||
from agenteval.models import RuleLogic
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class RuleOutcome:
|
||
"""单条规则的判定结果（引擎执行规则后规范化为此形状）。"""
|
||
|
||
passed: bool
|
||
score: Optional[float] = None
|
||
weight: float = 1.0
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class CaseOutcome:
|
||
"""单个用例的权威判定结果。"""
|
||
|
||
passed: bool
|
||
connectivity: bool
|
||
|
||
|
||
def combine_case_outcome(
|
||
*,
|
||
all_replied: bool,
|
||
explicit: Sequence[RuleOutcome] = (),
|
||
implicit: Sequence[RuleOutcome] = (),
|
||
rule_logic: RuleLogic = RuleLogic.ALL,
|
||
threshold: float = 0.6,
|
||
) -> CaseOutcome:
|
||
if not explicit and not implicit:
|
||
# 连通用例：connectivity 标记仅在连通成功时为真
|
||
ok = all_replied
|
||
return CaseOutcome(passed=ok, connectivity=ok)
|
||
|
||
if not explicit:
|
||
explicit_ok = True
|
||
elif rule_logic == RuleLogic.ANY:
|
||
explicit_ok = any(r.passed for r in explicit)
|
||
elif rule_logic == RuleLogic.WEIGHTED:
|
||
total_weight = sum(r.weight for r in explicit)
|
||
weighted_score = sum(
|
||
(r.score if r.score is not None else (1.0 if r.passed else 0.0)) * r.weight
|
||
for r in explicit
|
||
)
|
||
avg = weighted_score / total_weight if total_weight > 0 else 0.0
|
||
explicit_ok = avg >= threshold
|
||
else: # ALL and fallback
|
||
explicit_ok = all(r.passed for r in explicit)
|
||
|
||
implicit_ok = all(r.passed for r in implicit)
|
||
return CaseOutcome(passed=explicit_ok and implicit_ok, connectivity=False)
|