5db0ede4f4
Some checks failed
CI / test (push) Failing after 50s
「用例是否通过」此前散落 8 处且互相矛盾：engine 权威判定焊死在持久化里 不可单测；report 聚合/compare/markdown 各自从规则结果反推，规则还不一致 （markdown 用 all([]) 把故障用例误渲染成 ✅）。 - 新增纯函数 evaluation/judgement.combine_case_outcome（RuleOutcome/ CaseOutcome），判定组合脱离通道与 DB 可单测（判定矩阵 14 例） - engine 调用它一次，逐用例权威结果写入 summary.case_outcomes（JSON， 零迁移）；report/compare/markdown 只读权威值，老 run fallback 反推 - 故障用例判 False（ADR-0002）：修正 markdown 的 ✅ bug 与 compare 的 None；顺带修 engine 连通用例无回复也算通过的 bug - pass_rate 口径改为用例级（CONTEXT.md 词条），规则级保留在 passed_rules/total_rules；CLI 对比标签同步更正 - 修 RunRepository.update 漏拷 scenario_version/triggered_by 的字段漂移
130 lines
4.6 KiB
Python
130 lines
4.6 KiB
Python
"""判定组合器（judgement）单元测试 — 纯函数，无通道无 DB。
|
||
|
||
语义来源：CONTEXT.md（连通用例/通过率）、ADR-0002（故障=不通过）、
|
||
spec v0.5（期望叠加：隐式规则是 rule_logic 之外的硬约束）。
|
||
"""
|
||
|
||
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
|
||
from agenteval.models import RuleLogic
|
||
|
||
|
||
def _r(passed: bool, score: float | None = None, weight: float = 1.0) -> RuleOutcome:
|
||
return RuleOutcome(passed=passed, score=score, weight=weight)
|
||
|
||
|
||
# ── 连通用例（无任何判定标准） ──────────────────────────────────────
|
||
|
||
|
||
def test_connectivity_case_all_replied_passes():
|
||
outcome = combine_case_outcome(all_replied=True)
|
||
assert outcome == CaseOutcome(passed=True, connectivity=True)
|
||
|
||
|
||
def test_connectivity_case_missing_reply_fails():
|
||
# 无回复=故障=不通过（ADR-0002 服务视角），且不再标注为连通
|
||
outcome = combine_case_outcome(all_replied=False)
|
||
assert outcome == CaseOutcome(passed=False, connectivity=False)
|
||
|
||
|
||
# ── 显式规则组合（rule_logic） ─────────────────────────────────────
|
||
|
||
|
||
def test_all_logic_every_rule_passes():
|
||
outcome = combine_case_outcome(all_replied=True, explicit=[_r(True), _r(True)])
|
||
assert outcome == CaseOutcome(passed=True, connectivity=False)
|
||
|
||
|
||
def test_all_logic_single_failure_fails():
|
||
outcome = combine_case_outcome(all_replied=True, explicit=[_r(True), _r(False)])
|
||
assert outcome.passed is False
|
||
|
||
|
||
def test_any_logic_single_pass_suffices():
|
||
outcome = combine_case_outcome(
|
||
all_replied=True, explicit=[_r(False), _r(True)], rule_logic=RuleLogic.ANY
|
||
)
|
||
assert outcome.passed is True
|
||
|
||
|
||
def test_any_logic_no_pass_fails():
|
||
outcome = combine_case_outcome(
|
||
all_replied=True, explicit=[_r(False), _r(False)], rule_logic=RuleLogic.ANY
|
||
)
|
||
assert outcome.passed is False
|
||
|
||
|
||
def test_weighted_logic_above_threshold_passes():
|
||
outcome = combine_case_outcome(
|
||
all_replied=True,
|
||
explicit=[_r(True, score=0.9, weight=2.0), _r(False, score=0.3, weight=1.0)],
|
||
rule_logic=RuleLogic.WEIGHTED,
|
||
threshold=0.6,
|
||
)
|
||
# (0.9*2 + 0.3*1) / 3 = 0.7 >= 0.6
|
||
assert outcome.passed is True
|
||
|
||
|
||
def test_weighted_logic_below_threshold_fails():
|
||
outcome = combine_case_outcome(
|
||
all_replied=True,
|
||
explicit=[_r(True, score=0.9, weight=1.0), _r(False, score=0.1, weight=2.0)],
|
||
rule_logic=RuleLogic.WEIGHTED,
|
||
threshold=0.6,
|
||
)
|
||
# (0.9 + 0.1*2) / 3 ≈ 0.367 < 0.6
|
||
assert outcome.passed is False
|
||
|
||
|
||
def test_weighted_logic_scoreless_rule_uses_binary_score():
|
||
# 无 score 的规则按通过=1.0 / 失败=0.0 计（与引擎既有口径一致）
|
||
outcome = combine_case_outcome(
|
||
all_replied=True,
|
||
explicit=[_r(True, score=None, weight=1.0)],
|
||
rule_logic=RuleLogic.WEIGHTED,
|
||
threshold=0.6,
|
||
)
|
||
assert outcome.passed is True
|
||
|
||
|
||
# ── 期望叠加（隐式规则是硬约束） ───────────────────────────────────
|
||
|
||
|
||
def test_implicit_failure_vetoes_explicit_pass():
|
||
outcome = combine_case_outcome(
|
||
all_replied=True, explicit=[_r(True)], implicit=[_r(False)]
|
||
)
|
||
assert outcome.passed is False
|
||
|
||
|
||
def test_implicit_failure_vetoes_even_any_logic():
|
||
# 隐式规则不参与 rule_logic 组合：any 满足也救不回隐式失败
|
||
outcome = combine_case_outcome(
|
||
all_replied=True,
|
||
explicit=[_r(True), _r(False)],
|
||
implicit=[_r(False)],
|
||
rule_logic=RuleLogic.ANY,
|
||
)
|
||
assert outcome.passed is False
|
||
|
||
|
||
def test_implicit_only_case_passes_when_implicit_pass():
|
||
# 仅期望无显式规则：显式组合空真
|
||
outcome = combine_case_outcome(all_replied=True, implicit=[_r(True)])
|
||
assert outcome == CaseOutcome(passed=True, connectivity=False)
|
||
|
||
|
||
def test_explicit_failure_not_saved_by_implicit_pass():
|
||
outcome = combine_case_outcome(
|
||
all_replied=True, explicit=[_r(False)], implicit=[_r(True)]
|
||
)
|
||
assert outcome.passed is False
|
||
|
||
|
||
# ── 有规则用例的回复缺失：判定权在规则 ─────────────────────────────
|
||
|
||
|
||
def test_ruled_case_judged_by_rules_even_with_missing_reply():
|
||
# 有判定标准时由规则说了算（规则自身会因无回复而失败），不因缺回复直接判死
|
||
outcome = combine_case_outcome(all_replied=False, explicit=[_r(True)])
|
||
assert outcome == CaseOutcome(passed=True, connectivity=False)
|