refactor(judgement): converge case-pass decision into one deep module
Some checks failed
CI / test (push) Failing after 50s

「用例是否通过」此前散落 8 处且互相矛盾:engine 权威判定焊死在持久化里
不可单测;report 聚合/compare/markdown 各自从规则结果反推,规则还不一致
(markdown 用 all([]) 把故障用例误渲染成 )。

- 新增纯函数 evaluation/judgement.combine_case_outcome(RuleOutcome/
  CaseOutcome),判定组合脱离通道与 DB 可单测(判定矩阵 14 例)
- engine 调用它一次,逐用例权威结果写入 summary.case_outcomes(JSON,
  零迁移);report/compare/markdown 只读权威值,老 run fallback 反推
- 故障用例判 False(ADR-0002):修正 markdown 的  bug 与 compare 的
  None;顺带修 engine 连通用例无回复也算通过的 bug
- pass_rate 口径改为用例级(CONTEXT.md 词条),规则级保留在
  passed_rules/total_rules;CLI 对比标签同步更正
- 修 RunRepository.update 漏拷 scenario_version/triggered_by 的字段漂移
This commit is contained in:
sinohqb 2026-07-29 19:45:02 +08:00
parent f1aa61edd0
commit 5db0ede4f4
9 changed files with 366 additions and 70 deletions

View File

@ -12,6 +12,7 @@ from typing import Any, Callable, Optional
from agenteval.channels.base import EvalChannel
from agenteval.channels.factory import ChannelFactory
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
from agenteval.evaluation.rules import RuleResult, get_rule
from agenteval.model_gateway import ModelGateway
from agenteval.models import (
@ -22,7 +23,6 @@ from agenteval.models import (
EvalTarget,
ModelCapability,
ModelPurpose,
RuleLogic,
RunStatus,
RunTrigger,
Scenario,
@ -129,6 +129,7 @@ class EvalEngine:
total_cases = len(self.scenario.cases)
passed_cases = 0
failed_cases = 0
case_outcomes: dict[str, dict[str, bool]] = {}
for idx, case in enumerate(self.scenario.cases, start=1):
self._check_cancel()
@ -142,12 +143,13 @@ class EvalEngine:
},
)
async with self._case_semaphore:
case_passed, rule_pass, rule_total = await self._run_case(
outcome, rule_pass, rule_total = await self._run_case(
run,
case,
progress_callback,
)
if case_passed:
case_outcomes[case.id] = {"passed": outcome.passed, "connectivity": outcome.connectivity}
if outcome.passed:
passed_cases += 1
else:
failed_cases += 1
@ -158,7 +160,7 @@ class EvalEngine:
"index": idx,
"total": total_cases,
"case_id": case.id,
"passed": case_passed,
"passed": outcome.passed,
"rule_pass_count": rule_pass,
"rule_total": rule_total,
},
@ -174,7 +176,10 @@ class EvalEngine:
"failed_cases": failed_cases,
"total_rules": total_rules,
"passed_rules": passed_rules,
"pass_rate": round(passed_rules / total_rules, 4) if total_rules else 0.0,
# 通过率是用例级口径CONTEXT.md规则级数字保留在 passed_rules/total_rules
"pass_rate": round(passed_cases / total_cases, 4) if total_cases else 0.0,
# 逐用例权威判定judgement.py 算一次),报告/对比/渲染层只读不重算
"case_outcomes": case_outcomes,
}
# Surface fatal case-level errors (e.g. dynamic generation failures)
# so the report / DB record shows *why* a run produced no results.
@ -249,8 +254,9 @@ class EvalEngine:
run: EvalRun,
case: Case,
progress_callback: Optional[ProgressCallback],
) -> tuple[bool, int, int]:
"""Run a single case; returns (all_rules_passed, passed_rules, total_rules)."""
) -> tuple[CaseOutcome, int, int]:
"""Run a single case; returns (outcome, passed_rules, total_rules)."""
failed = CaseOutcome(passed=False, connectivity=False)
if case.type == CaseType.DYNAMIC:
generated = await self._generate_messages(case, progress_callback)
if not generated:
@ -262,7 +268,7 @@ class EvalEngine:
"case_id": case.id,
},
)
return False, 0, 0
return failed, 0, 0
case = case.model_copy(update={"messages": generated})
dialog: list[Turn] = []
@ -302,7 +308,7 @@ class EvalEngine:
"error": send_result.error,
},
)
return False, 0, 0
return failed, 0, 0
try:
reply = await self.channel.poll_reply(
@ -331,7 +337,7 @@ class EvalEngine:
"error": f"poll_reply 异常: {poll_exc}",
},
)
return False, 0, 0
return failed, 0, 0
received_at = utc_now()
latency_ms = None
@ -366,7 +372,7 @@ class EvalEngine:
)
if not dialog:
return False, 0, 0
return failed, 0, 0
return await self._save_rule_results(run, case, dialog[-1], dialog, progress_callback)
@ -377,14 +383,11 @@ class EvalEngine:
turn: Turn,
dialog: list[Turn],
progress_callback: Optional[ProgressCallback],
) -> tuple[bool, int, int]:
"""Apply rules and save results; returns (case_passed, passed_count, total_count).
) -> tuple[CaseOutcome, int, int]:
"""Apply rules and save results; returns (outcome, passed_count, total_count).
Judgement semantics (spec v0.5 / CONTEXT.md):
- Explicit rules are combined by case.rule_logic (ALL / ANY / WEIGHTED).
- Expectations always derive implicit checks, additive to explicit
rules. They are hard constraints: they never join the rule_logic
combination, and any implicit failure fails the case.
判定组合本身在 judgement.combine_case_outcome单一权威
本方法只负责执行规则持久化结果并把规则输出规范化为 RuleOutcome
"""
from agenteval.models import EvalRuleConfig
@ -411,17 +414,16 @@ class EvalEngine:
)
)
all_replied = bool(dialog) and all(t.reply is not None for t in dialog)
if not rules_config and not implicit_config:
# 连通用例:无任何判定标准,收到回复即通过
return True, 0, 0
# 连通用例:收到全部回复才通过(无回复=故障ADR-0002
return combine_case_outcome(all_replied=all_replied), 0, 0
passed_count = 0
total_count = 0
explicit_passed = 0
explicit_total = 0
weighted_score = 0.0
total_weight = 0.0
implicit_all_passed = True
explicit_outcomes: list[RuleOutcome] = []
implicit_outcomes: list[RuleOutcome] = []
all_rules = [(cfg, False) for cfg in rules_config] + [(cfg, True) for cfg in implicit_config]
for rule_config, is_implicit in all_rules:
@ -457,18 +459,11 @@ class EvalEngine:
if result.passed:
passed_count += 1
rule_outcome = RuleOutcome(passed=result.passed, score=result.score, weight=rule_config.weight)
if is_implicit:
if not result.passed:
implicit_all_passed = False
implicit_outcomes.append(rule_outcome)
else:
explicit_total += 1
if result.passed:
explicit_passed += 1
# Weighted scoring: use rule score (default 1.0 if passed, 0.0 if failed)
score_val = result.score if result.score is not None else (1.0 if result.passed else 0.0)
weight = rule_config.weight
weighted_score += score_val * weight
total_weight += weight
explicit_outcomes.append(rule_outcome)
await self._emit(
progress_callback,
@ -484,21 +479,14 @@ class EvalEngine:
},
)
# Combine explicit rules by rule_logic; no explicit rules → vacuously true.
logic = case.rule_logic
if not rules_config:
explicit_ok = True
elif logic == RuleLogic.ANY:
explicit_ok = explicit_passed > 0
elif logic == RuleLogic.WEIGHTED:
avg = weighted_score / total_weight if total_weight > 0 else 0.0
explicit_ok = avg >= case.rule_pass_threshold
else: # ALL and fallback
explicit_ok = explicit_passed == explicit_total
case_passed = explicit_ok and implicit_all_passed
return case_passed, passed_count, total_count
outcome = combine_case_outcome(
all_replied=all_replied,
explicit=explicit_outcomes,
implicit=implicit_outcomes,
rule_logic=case.rule_logic,
threshold=case.rule_pass_threshold,
)
return outcome, passed_count, total_count
async def _generate_messages(
self,

View File

@ -0,0 +1,64 @@
"""Case judgement — the single authority for "did this case pass".
判定语义CONTEXT.md / spec v0.5 / ADR-0002
- 连通用例无显式规则无期望每轮收到回复即通过缺回复=故障=不通过
- 显式规则按 rule_logicALL / ANY / WEIGHTED组合无显式规则时空真
- 期望派生的隐式规则是 rule_logic 之外的硬约束任一失败则用例失败
引擎执行时调用一次结果写入 run.summary["case_outcomes"]
报告 / 对比 / 渲染层只消费该权威值不得各自重算
"""
from dataclasses import dataclass
from typing import Optional, Sequence
from agenteval.models import RuleLogic
@dataclass(frozen=True)
class RuleOutcome:
"""单条规则的判定结果(引擎执行规则后规范化为此形状)。"""
passed: bool
score: Optional[float] = None
weight: float = 1.0
@dataclass(frozen=True)
class CaseOutcome:
"""单个用例的权威判定结果。"""
passed: bool
connectivity: bool
def combine_case_outcome(
*,
all_replied: bool,
explicit: Sequence[RuleOutcome] = (),
implicit: Sequence[RuleOutcome] = (),
rule_logic: RuleLogic = RuleLogic.ALL,
threshold: float = 0.6,
) -> CaseOutcome:
if not explicit and not implicit:
# 连通用例connectivity 标记仅在连通成功时为真
ok = all_replied
return CaseOutcome(passed=ok, connectivity=ok)
if not explicit:
explicit_ok = True
elif rule_logic == RuleLogic.ANY:
explicit_ok = any(r.passed for r in explicit)
elif rule_logic == RuleLogic.WEIGHTED:
total_weight = sum(r.weight for r in explicit)
weighted_score = sum(
(r.score if r.score is not None else (1.0 if r.passed else 0.0)) * r.weight
for r in explicit
)
avg = weighted_score / total_weight if total_weight > 0 else 0.0
explicit_ok = avg >= threshold
else: # ALL and fallback
explicit_ok = all(r.passed for r in explicit)
implicit_ok = all(r.passed for r in implicit)
return CaseOutcome(passed=explicit_ok and implicit_ok, connectivity=False)

View File

@ -139,20 +139,36 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
summary = run.summary or {}
errored_case_ids = {e.get("case_id") for e in summary.get("case_errors", [])}
# 权威判定:引擎经 judgement.combine_case_outcome 算一次写入 summary
# 老 run 没有该字段时退回从持久化结果反推WEIGHTED/ANY 只能近似)。
authoritative: dict[str, Any] = summary.get("case_outcomes", {})
cases = []
for case_id in sorted(case_map.keys()):
item = case_map[case_id]
# 连通用例无任何判定结果且每轮都收到回复、无用例级错误CONTEXT.md
connectivity = (
not item["results"]
and bool(item["turns"])
and item["all_replied"]
and case_id not in errored_case_ids
)
if case_id in authoritative:
outcome = authoritative[case_id]
connectivity = bool(outcome.get("connectivity"))
passed = bool(outcome.get("passed"))
else:
# 连通用例无任何判定结果且每轮都收到回复、无用例级错误CONTEXT.md
connectivity = (
not item["results"]
and bool(item["turns"])
and item["all_replied"]
and case_id not in errored_case_ids
)
if connectivity:
passed = True
elif not item["results"]:
# 故障用例无结果且非连通不通过ADR-0002
passed = False
else:
passed = all(r["passed"] for r in item["results"])
cases.append(
{
"case_id": case_id,
"passed": passed,
"connectivity": connectivity,
"turns": sorted(item["turns"], key=lambda x: x["round"]),
"results": item["results"],
@ -221,16 +237,8 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[
cb = cases_b.get(cid)
def _case_passed(c):
if not c:
return None
if c.get("connectivity"):
# 连通用例收到回复即通过(引擎口径)
return True
results = c.get("results", [])
if not results:
# No rule results (e.g. errored case) must not count as passed.
return None
return all(r["passed"] for r in results)
# None 仅表示该 run 没有这个用例;判定本身读权威 passed 字段
return c["passed"] if c else None
case_diffs.append(
{
@ -311,8 +319,7 @@ def render_markdown_report(run_id: str, session=None) -> str:
if case.get("connectivity"):
badge = "🔗"
else:
case_passed = all(r["passed"] for r in case.get("results", []))
badge = "" if case_passed else ""
badge = "" if case.get("passed") else ""
title = f"### {badge} 用例 `{case['case_id']}`"
if case.get("connectivity"):
title += "(连通用例,未配置判定标准)"

View File

@ -302,7 +302,9 @@ class RunRepository:
return None
existing.target_id = run.target_id
existing.scenario_id = run.scenario_id
existing.scenario_version = run.scenario_version
existing.status = run.status.value
existing.triggered_by = run.triggered_by.value
existing.completed_at = run.completed_at
if run.summary:
existing.set_summary(run.summary)

View File

@ -65,4 +65,4 @@ def compare_reports(
console.print(f"对比: {run_id_1} vs {run_id_2}")
console.print(f" 用例数: {report1['summary']['total_cases']} -> {report2['summary']['total_cases']}")
console.print(f" 通过用例: {report1['summary']['passed_cases']} -> {report2['summary']['passed_cases']}")
console.print(f" 规则通过率: {report1['summary']['pass_rate']:.2%} -> {report2['summary']['pass_rate']:.2%}")
console.print(f" 通过率: {report1['summary']['pass_rate']:.2%} -> {report2['summary']['pass_rate']:.2%}")

View File

@ -455,3 +455,42 @@ async def test_engine_run_snapshots_scenario_version(db_session):
persisted = RunRepository(db_session).get(run.id)
assert persisted.scenario_version == 3
# ── 权威判定写入 summary.case_outcomes判定语义收敛 ────────────────────
async def test_summary_contains_case_outcomes_and_case_level_pass_rate(db_session):
scenario = Scenario(id="s-1", name="outcomes", cases=[
# 连通用例(无规则无期望)
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
# 判定失败用例(期望不满足)
Case(id="bad", type=CaseType.SINGLE, messages=["hi"],
expectations=Expectation(keywords_include=["__NOPE__"])),
])
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
run = await engine.run()
outcomes = run.summary["case_outcomes"]
assert outcomes["conn"] == {"passed": True, "connectivity": True}
assert outcomes["bad"] == {"passed": False, "connectivity": False}
# 通过率为用例级口径CONTEXT.md不再是规则级
assert run.summary["pass_rate"] == 0.5
async def test_connectivity_case_without_reply_fails(db_session):
"""连通用例没收到回复=故障=不通过(此前无条件判通过的 bug"""
scenario = Scenario(id="s-1", name="conn-fail", cases=[
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
])
channel = MockChannel(missing_reply=True)
engine = _build_engine(
scenario, channel, session=db_session,
timeout_config=TimeoutConfig(poll_reply=0.2),
)
run = await engine.run()
assert run.summary["passed_cases"] == 0
assert run.summary["case_outcomes"]["conn"] == {"passed": False, "connectivity": False}

View File

@ -0,0 +1,129 @@
"""判定组合器judgement单元测试 — 纯函数,无通道无 DB。
语义来源CONTEXT.md连通用例/通过率ADR-0002故障=不通过
spec v0.5期望叠加隐式规则是 rule_logic 之外的硬约束
"""
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
from agenteval.models import RuleLogic
def _r(passed: bool, score: float | None = None, weight: float = 1.0) -> RuleOutcome:
return RuleOutcome(passed=passed, score=score, weight=weight)
# ── 连通用例(无任何判定标准) ──────────────────────────────────────
def test_connectivity_case_all_replied_passes():
outcome = combine_case_outcome(all_replied=True)
assert outcome == CaseOutcome(passed=True, connectivity=True)
def test_connectivity_case_missing_reply_fails():
# 无回复=故障=不通过ADR-0002 服务视角),且不再标注为连通
outcome = combine_case_outcome(all_replied=False)
assert outcome == CaseOutcome(passed=False, connectivity=False)
# ── 显式规则组合rule_logic ─────────────────────────────────────
def test_all_logic_every_rule_passes():
outcome = combine_case_outcome(all_replied=True, explicit=[_r(True), _r(True)])
assert outcome == CaseOutcome(passed=True, connectivity=False)
def test_all_logic_single_failure_fails():
outcome = combine_case_outcome(all_replied=True, explicit=[_r(True), _r(False)])
assert outcome.passed is False
def test_any_logic_single_pass_suffices():
outcome = combine_case_outcome(
all_replied=True, explicit=[_r(False), _r(True)], rule_logic=RuleLogic.ANY
)
assert outcome.passed is True
def test_any_logic_no_pass_fails():
outcome = combine_case_outcome(
all_replied=True, explicit=[_r(False), _r(False)], rule_logic=RuleLogic.ANY
)
assert outcome.passed is False
def test_weighted_logic_above_threshold_passes():
outcome = combine_case_outcome(
all_replied=True,
explicit=[_r(True, score=0.9, weight=2.0), _r(False, score=0.3, weight=1.0)],
rule_logic=RuleLogic.WEIGHTED,
threshold=0.6,
)
# (0.9*2 + 0.3*1) / 3 = 0.7 >= 0.6
assert outcome.passed is True
def test_weighted_logic_below_threshold_fails():
outcome = combine_case_outcome(
all_replied=True,
explicit=[_r(True, score=0.9, weight=1.0), _r(False, score=0.1, weight=2.0)],
rule_logic=RuleLogic.WEIGHTED,
threshold=0.6,
)
# (0.9 + 0.1*2) / 3 ≈ 0.367 < 0.6
assert outcome.passed is False
def test_weighted_logic_scoreless_rule_uses_binary_score():
# 无 score 的规则按通过=1.0 / 失败=0.0 计(与引擎既有口径一致)
outcome = combine_case_outcome(
all_replied=True,
explicit=[_r(True, score=None, weight=1.0)],
rule_logic=RuleLogic.WEIGHTED,
threshold=0.6,
)
assert outcome.passed is True
# ── 期望叠加(隐式规则是硬约束) ───────────────────────────────────
def test_implicit_failure_vetoes_explicit_pass():
outcome = combine_case_outcome(
all_replied=True, explicit=[_r(True)], implicit=[_r(False)]
)
assert outcome.passed is False
def test_implicit_failure_vetoes_even_any_logic():
# 隐式规则不参与 rule_logic 组合any 满足也救不回隐式失败
outcome = combine_case_outcome(
all_replied=True,
explicit=[_r(True), _r(False)],
implicit=[_r(False)],
rule_logic=RuleLogic.ANY,
)
assert outcome.passed is False
def test_implicit_only_case_passes_when_implicit_pass():
# 仅期望无显式规则:显式组合空真
outcome = combine_case_outcome(all_replied=True, implicit=[_r(True)])
assert outcome == CaseOutcome(passed=True, connectivity=False)
def test_explicit_failure_not_saved_by_implicit_pass():
outcome = combine_case_outcome(
all_replied=True, explicit=[_r(False)], implicit=[_r(True)]
)
assert outcome.passed is False
# ── 有规则用例的回复缺失:判定权在规则 ─────────────────────────────
def test_ruled_case_judged_by_rules_even_with_missing_reply():
# 有判定标准时由规则说了算(规则自身会因无回复而失败),不因缺回复直接判死
outcome = combine_case_outcome(all_replied=False, explicit=[_r(True)])
assert outcome == CaseOutcome(passed=True, connectivity=False)

View File

@ -36,3 +36,21 @@ def test_mark_orphans_failed_noop_when_clean(db_session):
repo = RunRepository(db_session)
_make_run(db_session, RunStatus.COMPLETED)
assert repo.mark_orphans_failed() == 0
def test_update_preserves_scenario_version_and_triggered_by(db_session):
"""update() 不得丢字段scenario_version / triggered_by 必须回写(漂移回归)。"""
from agenteval.models import RunTrigger
repo = RunRepository(db_session)
run = repo.create(EvalRun(
target_id="t-1", scenario_id="s-1", status=RunStatus.RUNNING,
scenario_version=4, triggered_by=RunTrigger.AI_ASSISTANT,
))
run.scenario_version = 5
run.triggered_by = RunTrigger.CLI
run.status = RunStatus.COMPLETED
updated = repo.update(run)
assert updated.scenario_version == 5
assert updated.triggered_by == RunTrigger.CLI

View File

@ -361,3 +361,52 @@ def test_render_json_report_is_valid_json(report_session):
json_text = render_json_report(run_id, report_session)
parsed = json.loads(json_text)
assert parsed["run_id"] == run_id
# ── 权威判定消费judgement 语义收敛) ────────────────────────────────────
def test_case_dict_contains_passed_fallback(report_session):
"""老 run无 case_outcomespassed 反推得出,故障用例=FalseADR-0002"""
run_id = _seed_run(report_session, pass_rate=0.5, n_cases=2,
connectivity_cases=1, errored_cases=1)
report = generate_report(run_id, report_session)
by_id = {c["case_id"]: c for c in report["cases"]}
assert by_id["c0"]["passed"] is True
assert by_id["c1"]["passed"] is False
assert by_id["conn0"]["passed"] is True
assert by_id["err0"]["passed"] is False
def test_markdown_errored_case_shows_failed_badge(report_session):
"""故障用例(无结果且非连通)在 MD 中必须 ❌ —— 此前 all([]) 误判 ✅。"""
run_id = _seed_run(report_session, n_cases=1, errored_cases=1)
md = render_markdown_report(run_id, report_session)
assert "❌ 用例 `err0`" in md
assert "✅ 用例 `err0`" not in md
def test_authoritative_case_outcomes_override_reconstruction(report_session):
"""summary.case_outcomes 是权威判定:与规则结果反推冲突时以权威为准。"""
run_id = _seed_run(report_session, pass_rate=1.0, n_cases=1)
repo = RunRepository(report_session)
run = repo.get(run_id)
# 模拟 weighted 阈值未达:规则单条通过但用例判失败(反推 all() 会误判 True
run.summary = {**run.summary, "case_outcomes": {"c0": {"passed": False, "connectivity": False}}}
repo.update(run)
report = generate_report(run_id, report_session)
assert report["cases"][0]["passed"] is False
md = render_markdown_report(run_id, report_session)
assert "❌ 用例 `c0`" in md
def test_compare_errored_case_counts_as_failed(report_session):
"""对比中故障用例判 False 而非 NoneADR-0002 服务视角)。"""
run_id_a = _seed_run(report_session, n_cases=1, errored_cases=1)
sid = _scenario_of(report_session, run_id_a)
run_id_b = _seed_run(report_session, n_cases=1, errored_cases=1, scenario_id=sid)
result = generate_compare_report(run_id_a, run_id_b, report_session)
by_id = {c["case_id"]: c for c in result["cases"]}
assert by_id["err0"]["run_a_passed"] is False
assert by_id["err0"]["run_b_passed"] is False
assert by_id["err0"]["changed"] is False