refactor(judgement): converge case-pass decision into one deep module
Some checks failed
CI / test (push) Failing after 50s
Some checks failed
CI / test (push) Failing after 50s
「用例是否通过」此前散落 8 处且互相矛盾:engine 权威判定焊死在持久化里 不可单测;report 聚合/compare/markdown 各自从规则结果反推,规则还不一致 (markdown 用 all([]) 把故障用例误渲染成 ✅)。 - 新增纯函数 evaluation/judgement.combine_case_outcome(RuleOutcome/ CaseOutcome),判定组合脱离通道与 DB 可单测(判定矩阵 14 例) - engine 调用它一次,逐用例权威结果写入 summary.case_outcomes(JSON, 零迁移);report/compare/markdown 只读权威值,老 run fallback 反推 - 故障用例判 False(ADR-0002):修正 markdown 的 ✅ bug 与 compare 的 None;顺带修 engine 连通用例无回复也算通过的 bug - pass_rate 口径改为用例级(CONTEXT.md 词条),规则级保留在 passed_rules/total_rules;CLI 对比标签同步更正 - 修 RunRepository.update 漏拷 scenario_version/triggered_by 的字段漂移
This commit is contained in:
parent
f1aa61edd0
commit
5db0ede4f4
@ -12,6 +12,7 @@ from typing import Any, Callable, Optional
|
||||
|
||||
from agenteval.channels.base import EvalChannel
|
||||
from agenteval.channels.factory import ChannelFactory
|
||||
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
|
||||
from agenteval.evaluation.rules import RuleResult, get_rule
|
||||
from agenteval.model_gateway import ModelGateway
|
||||
from agenteval.models import (
|
||||
@ -22,7 +23,6 @@ from agenteval.models import (
|
||||
EvalTarget,
|
||||
ModelCapability,
|
||||
ModelPurpose,
|
||||
RuleLogic,
|
||||
RunStatus,
|
||||
RunTrigger,
|
||||
Scenario,
|
||||
@ -129,6 +129,7 @@ class EvalEngine:
|
||||
total_cases = len(self.scenario.cases)
|
||||
passed_cases = 0
|
||||
failed_cases = 0
|
||||
case_outcomes: dict[str, dict[str, bool]] = {}
|
||||
|
||||
for idx, case in enumerate(self.scenario.cases, start=1):
|
||||
self._check_cancel()
|
||||
@ -142,12 +143,13 @@ class EvalEngine:
|
||||
},
|
||||
)
|
||||
async with self._case_semaphore:
|
||||
case_passed, rule_pass, rule_total = await self._run_case(
|
||||
outcome, rule_pass, rule_total = await self._run_case(
|
||||
run,
|
||||
case,
|
||||
progress_callback,
|
||||
)
|
||||
if case_passed:
|
||||
case_outcomes[case.id] = {"passed": outcome.passed, "connectivity": outcome.connectivity}
|
||||
if outcome.passed:
|
||||
passed_cases += 1
|
||||
else:
|
||||
failed_cases += 1
|
||||
@ -158,7 +160,7 @@ class EvalEngine:
|
||||
"index": idx,
|
||||
"total": total_cases,
|
||||
"case_id": case.id,
|
||||
"passed": case_passed,
|
||||
"passed": outcome.passed,
|
||||
"rule_pass_count": rule_pass,
|
||||
"rule_total": rule_total,
|
||||
},
|
||||
@ -174,7 +176,10 @@ class EvalEngine:
|
||||
"failed_cases": failed_cases,
|
||||
"total_rules": total_rules,
|
||||
"passed_rules": passed_rules,
|
||||
"pass_rate": round(passed_rules / total_rules, 4) if total_rules else 0.0,
|
||||
# 通过率是用例级口径(CONTEXT.md);规则级数字保留在 passed_rules/total_rules
|
||||
"pass_rate": round(passed_cases / total_cases, 4) if total_cases else 0.0,
|
||||
# 逐用例权威判定(judgement.py 算一次),报告/对比/渲染层只读不重算
|
||||
"case_outcomes": case_outcomes,
|
||||
}
|
||||
# Surface fatal case-level errors (e.g. dynamic generation failures)
|
||||
# so the report / DB record shows *why* a run produced no results.
|
||||
@ -249,8 +254,9 @@ class EvalEngine:
|
||||
run: EvalRun,
|
||||
case: Case,
|
||||
progress_callback: Optional[ProgressCallback],
|
||||
) -> tuple[bool, int, int]:
|
||||
"""Run a single case; returns (all_rules_passed, passed_rules, total_rules)."""
|
||||
) -> tuple[CaseOutcome, int, int]:
|
||||
"""Run a single case; returns (outcome, passed_rules, total_rules)."""
|
||||
failed = CaseOutcome(passed=False, connectivity=False)
|
||||
if case.type == CaseType.DYNAMIC:
|
||||
generated = await self._generate_messages(case, progress_callback)
|
||||
if not generated:
|
||||
@ -262,7 +268,7 @@ class EvalEngine:
|
||||
"case_id": case.id,
|
||||
},
|
||||
)
|
||||
return False, 0, 0
|
||||
return failed, 0, 0
|
||||
case = case.model_copy(update={"messages": generated})
|
||||
|
||||
dialog: list[Turn] = []
|
||||
@ -302,7 +308,7 @@ class EvalEngine:
|
||||
"error": send_result.error,
|
||||
},
|
||||
)
|
||||
return False, 0, 0
|
||||
return failed, 0, 0
|
||||
|
||||
try:
|
||||
reply = await self.channel.poll_reply(
|
||||
@ -331,7 +337,7 @@ class EvalEngine:
|
||||
"error": f"poll_reply 异常: {poll_exc}",
|
||||
},
|
||||
)
|
||||
return False, 0, 0
|
||||
return failed, 0, 0
|
||||
|
||||
received_at = utc_now()
|
||||
latency_ms = None
|
||||
@ -366,7 +372,7 @@ class EvalEngine:
|
||||
)
|
||||
|
||||
if not dialog:
|
||||
return False, 0, 0
|
||||
return failed, 0, 0
|
||||
|
||||
return await self._save_rule_results(run, case, dialog[-1], dialog, progress_callback)
|
||||
|
||||
@ -377,14 +383,11 @@ class EvalEngine:
|
||||
turn: Turn,
|
||||
dialog: list[Turn],
|
||||
progress_callback: Optional[ProgressCallback],
|
||||
) -> tuple[bool, int, int]:
|
||||
"""Apply rules and save results; returns (case_passed, passed_count, total_count).
|
||||
) -> tuple[CaseOutcome, int, int]:
|
||||
"""Apply rules and save results; returns (outcome, passed_count, total_count).
|
||||
|
||||
Judgement semantics (spec v0.5 / CONTEXT.md):
|
||||
- Explicit rules are combined by case.rule_logic (ALL / ANY / WEIGHTED).
|
||||
- Expectations always derive implicit checks, additive to explicit
|
||||
rules. They are hard constraints: they never join the rule_logic
|
||||
combination, and any implicit failure fails the case.
|
||||
判定组合本身在 judgement.combine_case_outcome(单一权威)——
|
||||
本方法只负责执行规则、持久化结果并把规则输出规范化为 RuleOutcome。
|
||||
"""
|
||||
from agenteval.models import EvalRuleConfig
|
||||
|
||||
@ -411,17 +414,16 @@ class EvalEngine:
|
||||
)
|
||||
)
|
||||
|
||||
all_replied = bool(dialog) and all(t.reply is not None for t in dialog)
|
||||
|
||||
if not rules_config and not implicit_config:
|
||||
# 连通用例:无任何判定标准,收到回复即通过
|
||||
return True, 0, 0
|
||||
# 连通用例:收到全部回复才通过(无回复=故障,ADR-0002)
|
||||
return combine_case_outcome(all_replied=all_replied), 0, 0
|
||||
|
||||
passed_count = 0
|
||||
total_count = 0
|
||||
explicit_passed = 0
|
||||
explicit_total = 0
|
||||
weighted_score = 0.0
|
||||
total_weight = 0.0
|
||||
implicit_all_passed = True
|
||||
explicit_outcomes: list[RuleOutcome] = []
|
||||
implicit_outcomes: list[RuleOutcome] = []
|
||||
|
||||
all_rules = [(cfg, False) for cfg in rules_config] + [(cfg, True) for cfg in implicit_config]
|
||||
for rule_config, is_implicit in all_rules:
|
||||
@ -457,18 +459,11 @@ class EvalEngine:
|
||||
if result.passed:
|
||||
passed_count += 1
|
||||
|
||||
rule_outcome = RuleOutcome(passed=result.passed, score=result.score, weight=rule_config.weight)
|
||||
if is_implicit:
|
||||
if not result.passed:
|
||||
implicit_all_passed = False
|
||||
implicit_outcomes.append(rule_outcome)
|
||||
else:
|
||||
explicit_total += 1
|
||||
if result.passed:
|
||||
explicit_passed += 1
|
||||
# Weighted scoring: use rule score (default 1.0 if passed, 0.0 if failed)
|
||||
score_val = result.score if result.score is not None else (1.0 if result.passed else 0.0)
|
||||
weight = rule_config.weight
|
||||
weighted_score += score_val * weight
|
||||
total_weight += weight
|
||||
explicit_outcomes.append(rule_outcome)
|
||||
|
||||
await self._emit(
|
||||
progress_callback,
|
||||
@ -484,21 +479,14 @@ class EvalEngine:
|
||||
},
|
||||
)
|
||||
|
||||
# Combine explicit rules by rule_logic; no explicit rules → vacuously true.
|
||||
logic = case.rule_logic
|
||||
if not rules_config:
|
||||
explicit_ok = True
|
||||
elif logic == RuleLogic.ANY:
|
||||
explicit_ok = explicit_passed > 0
|
||||
elif logic == RuleLogic.WEIGHTED:
|
||||
avg = weighted_score / total_weight if total_weight > 0 else 0.0
|
||||
explicit_ok = avg >= case.rule_pass_threshold
|
||||
else: # ALL and fallback
|
||||
explicit_ok = explicit_passed == explicit_total
|
||||
|
||||
case_passed = explicit_ok and implicit_all_passed
|
||||
|
||||
return case_passed, passed_count, total_count
|
||||
outcome = combine_case_outcome(
|
||||
all_replied=all_replied,
|
||||
explicit=explicit_outcomes,
|
||||
implicit=implicit_outcomes,
|
||||
rule_logic=case.rule_logic,
|
||||
threshold=case.rule_pass_threshold,
|
||||
)
|
||||
return outcome, passed_count, total_count
|
||||
|
||||
async def _generate_messages(
|
||||
self,
|
||||
|
||||
64
backend/agenteval/evaluation/judgement.py
Normal file
64
backend/agenteval/evaluation/judgement.py
Normal file
@ -0,0 +1,64 @@
|
||||
"""Case judgement — the single authority for "did this case pass".
|
||||
|
||||
判定语义(CONTEXT.md / spec v0.5 / ADR-0002):
|
||||
- 连通用例(无显式规则、无期望):每轮收到回复即通过;缺回复=故障=不通过。
|
||||
- 显式规则按 rule_logic(ALL / ANY / WEIGHTED)组合;无显式规则时空真。
|
||||
- 期望派生的隐式规则是 rule_logic 之外的硬约束,任一失败则用例失败。
|
||||
|
||||
引擎执行时调用一次,结果写入 run.summary["case_outcomes"],
|
||||
报告 / 对比 / 渲染层只消费该权威值,不得各自重算。
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Sequence
|
||||
|
||||
from agenteval.models import RuleLogic
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuleOutcome:
|
||||
"""单条规则的判定结果(引擎执行规则后规范化为此形状)。"""
|
||||
|
||||
passed: bool
|
||||
score: Optional[float] = None
|
||||
weight: float = 1.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CaseOutcome:
|
||||
"""单个用例的权威判定结果。"""
|
||||
|
||||
passed: bool
|
||||
connectivity: bool
|
||||
|
||||
|
||||
def combine_case_outcome(
|
||||
*,
|
||||
all_replied: bool,
|
||||
explicit: Sequence[RuleOutcome] = (),
|
||||
implicit: Sequence[RuleOutcome] = (),
|
||||
rule_logic: RuleLogic = RuleLogic.ALL,
|
||||
threshold: float = 0.6,
|
||||
) -> CaseOutcome:
|
||||
if not explicit and not implicit:
|
||||
# 连通用例:connectivity 标记仅在连通成功时为真
|
||||
ok = all_replied
|
||||
return CaseOutcome(passed=ok, connectivity=ok)
|
||||
|
||||
if not explicit:
|
||||
explicit_ok = True
|
||||
elif rule_logic == RuleLogic.ANY:
|
||||
explicit_ok = any(r.passed for r in explicit)
|
||||
elif rule_logic == RuleLogic.WEIGHTED:
|
||||
total_weight = sum(r.weight for r in explicit)
|
||||
weighted_score = sum(
|
||||
(r.score if r.score is not None else (1.0 if r.passed else 0.0)) * r.weight
|
||||
for r in explicit
|
||||
)
|
||||
avg = weighted_score / total_weight if total_weight > 0 else 0.0
|
||||
explicit_ok = avg >= threshold
|
||||
else: # ALL and fallback
|
||||
explicit_ok = all(r.passed for r in explicit)
|
||||
|
||||
implicit_ok = all(r.passed for r in implicit)
|
||||
return CaseOutcome(passed=explicit_ok and implicit_ok, connectivity=False)
|
||||
@ -139,10 +139,18 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
|
||||
|
||||
summary = run.summary or {}
|
||||
errored_case_ids = {e.get("case_id") for e in summary.get("case_errors", [])}
|
||||
# 权威判定:引擎经 judgement.combine_case_outcome 算一次写入 summary;
|
||||
# 老 run 没有该字段时退回从持久化结果反推(WEIGHTED/ANY 只能近似)。
|
||||
authoritative: dict[str, Any] = summary.get("case_outcomes", {})
|
||||
|
||||
cases = []
|
||||
for case_id in sorted(case_map.keys()):
|
||||
item = case_map[case_id]
|
||||
if case_id in authoritative:
|
||||
outcome = authoritative[case_id]
|
||||
connectivity = bool(outcome.get("connectivity"))
|
||||
passed = bool(outcome.get("passed"))
|
||||
else:
|
||||
# 连通用例:无任何判定结果,且每轮都收到回复、无用例级错误(CONTEXT.md)
|
||||
connectivity = (
|
||||
not item["results"]
|
||||
@ -150,9 +158,17 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
|
||||
and item["all_replied"]
|
||||
and case_id not in errored_case_ids
|
||||
)
|
||||
if connectivity:
|
||||
passed = True
|
||||
elif not item["results"]:
|
||||
# 故障用例(无结果且非连通)=不通过(ADR-0002)
|
||||
passed = False
|
||||
else:
|
||||
passed = all(r["passed"] for r in item["results"])
|
||||
cases.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"passed": passed,
|
||||
"connectivity": connectivity,
|
||||
"turns": sorted(item["turns"], key=lambda x: x["round"]),
|
||||
"results": item["results"],
|
||||
@ -221,16 +237,8 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[
|
||||
cb = cases_b.get(cid)
|
||||
|
||||
def _case_passed(c):
|
||||
if not c:
|
||||
return None
|
||||
if c.get("connectivity"):
|
||||
# 连通用例收到回复即通过(引擎口径)
|
||||
return True
|
||||
results = c.get("results", [])
|
||||
if not results:
|
||||
# No rule results (e.g. errored case) must not count as passed.
|
||||
return None
|
||||
return all(r["passed"] for r in results)
|
||||
# None 仅表示该 run 没有这个用例;判定本身读权威 passed 字段
|
||||
return c["passed"] if c else None
|
||||
|
||||
case_diffs.append(
|
||||
{
|
||||
@ -311,8 +319,7 @@ def render_markdown_report(run_id: str, session=None) -> str:
|
||||
if case.get("connectivity"):
|
||||
badge = "🔗"
|
||||
else:
|
||||
case_passed = all(r["passed"] for r in case.get("results", []))
|
||||
badge = "✅" if case_passed else "❌"
|
||||
badge = "✅" if case.get("passed") else "❌"
|
||||
title = f"### {badge} 用例 `{case['case_id']}`"
|
||||
if case.get("connectivity"):
|
||||
title += "(连通用例,未配置判定标准)"
|
||||
|
||||
@ -302,7 +302,9 @@ class RunRepository:
|
||||
return None
|
||||
existing.target_id = run.target_id
|
||||
existing.scenario_id = run.scenario_id
|
||||
existing.scenario_version = run.scenario_version
|
||||
existing.status = run.status.value
|
||||
existing.triggered_by = run.triggered_by.value
|
||||
existing.completed_at = run.completed_at
|
||||
if run.summary:
|
||||
existing.set_summary(run.summary)
|
||||
|
||||
@ -65,4 +65,4 @@ def compare_reports(
|
||||
console.print(f"对比: {run_id_1} vs {run_id_2}")
|
||||
console.print(f" 用例数: {report1['summary']['total_cases']} -> {report2['summary']['total_cases']}")
|
||||
console.print(f" 通过用例: {report1['summary']['passed_cases']} -> {report2['summary']['passed_cases']}")
|
||||
console.print(f" 规则通过率: {report1['summary']['pass_rate']:.2%} -> {report2['summary']['pass_rate']:.2%}")
|
||||
console.print(f" 通过率: {report1['summary']['pass_rate']:.2%} -> {report2['summary']['pass_rate']:.2%}")
|
||||
|
||||
@ -455,3 +455,42 @@ async def test_engine_run_snapshots_scenario_version(db_session):
|
||||
persisted = RunRepository(db_session).get(run.id)
|
||||
assert persisted.scenario_version == 3
|
||||
|
||||
|
||||
|
||||
# ── 权威判定写入 summary.case_outcomes(判定语义收敛) ────────────────────
|
||||
|
||||
async def test_summary_contains_case_outcomes_and_case_level_pass_rate(db_session):
|
||||
scenario = Scenario(id="s-1", name="outcomes", cases=[
|
||||
# 连通用例(无规则无期望)
|
||||
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
|
||||
# 判定失败用例(期望不满足)
|
||||
Case(id="bad", type=CaseType.SINGLE, messages=["hi"],
|
||||
expectations=Expectation(keywords_include=["__NOPE__"])),
|
||||
])
|
||||
channel = MockChannel()
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
|
||||
run = await engine.run()
|
||||
|
||||
outcomes = run.summary["case_outcomes"]
|
||||
assert outcomes["conn"] == {"passed": True, "connectivity": True}
|
||||
assert outcomes["bad"] == {"passed": False, "connectivity": False}
|
||||
# 通过率为用例级口径(CONTEXT.md),不再是规则级
|
||||
assert run.summary["pass_rate"] == 0.5
|
||||
|
||||
|
||||
async def test_connectivity_case_without_reply_fails(db_session):
|
||||
"""连通用例没收到回复=故障=不通过(此前无条件判通过的 bug)。"""
|
||||
scenario = Scenario(id="s-1", name="conn-fail", cases=[
|
||||
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
|
||||
])
|
||||
channel = MockChannel(missing_reply=True)
|
||||
engine = _build_engine(
|
||||
scenario, channel, session=db_session,
|
||||
timeout_config=TimeoutConfig(poll_reply=0.2),
|
||||
)
|
||||
|
||||
run = await engine.run()
|
||||
|
||||
assert run.summary["passed_cases"] == 0
|
||||
assert run.summary["case_outcomes"]["conn"] == {"passed": False, "connectivity": False}
|
||||
|
||||
129
tests/unit/test_judgement.py
Normal file
129
tests/unit/test_judgement.py
Normal file
@ -0,0 +1,129 @@
|
||||
"""判定组合器(judgement)单元测试 — 纯函数,无通道无 DB。
|
||||
|
||||
语义来源:CONTEXT.md(连通用例/通过率)、ADR-0002(故障=不通过)、
|
||||
spec v0.5(期望叠加:隐式规则是 rule_logic 之外的硬约束)。
|
||||
"""
|
||||
|
||||
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
|
||||
from agenteval.models import RuleLogic
|
||||
|
||||
|
||||
def _r(passed: bool, score: float | None = None, weight: float = 1.0) -> RuleOutcome:
|
||||
return RuleOutcome(passed=passed, score=score, weight=weight)
|
||||
|
||||
|
||||
# ── 连通用例(无任何判定标准) ──────────────────────────────────────
|
||||
|
||||
|
||||
def test_connectivity_case_all_replied_passes():
|
||||
outcome = combine_case_outcome(all_replied=True)
|
||||
assert outcome == CaseOutcome(passed=True, connectivity=True)
|
||||
|
||||
|
||||
def test_connectivity_case_missing_reply_fails():
|
||||
# 无回复=故障=不通过(ADR-0002 服务视角),且不再标注为连通
|
||||
outcome = combine_case_outcome(all_replied=False)
|
||||
assert outcome == CaseOutcome(passed=False, connectivity=False)
|
||||
|
||||
|
||||
# ── 显式规则组合(rule_logic) ─────────────────────────────────────
|
||||
|
||||
|
||||
def test_all_logic_every_rule_passes():
|
||||
outcome = combine_case_outcome(all_replied=True, explicit=[_r(True), _r(True)])
|
||||
assert outcome == CaseOutcome(passed=True, connectivity=False)
|
||||
|
||||
|
||||
def test_all_logic_single_failure_fails():
|
||||
outcome = combine_case_outcome(all_replied=True, explicit=[_r(True), _r(False)])
|
||||
assert outcome.passed is False
|
||||
|
||||
|
||||
def test_any_logic_single_pass_suffices():
|
||||
outcome = combine_case_outcome(
|
||||
all_replied=True, explicit=[_r(False), _r(True)], rule_logic=RuleLogic.ANY
|
||||
)
|
||||
assert outcome.passed is True
|
||||
|
||||
|
||||
def test_any_logic_no_pass_fails():
|
||||
outcome = combine_case_outcome(
|
||||
all_replied=True, explicit=[_r(False), _r(False)], rule_logic=RuleLogic.ANY
|
||||
)
|
||||
assert outcome.passed is False
|
||||
|
||||
|
||||
def test_weighted_logic_above_threshold_passes():
|
||||
outcome = combine_case_outcome(
|
||||
all_replied=True,
|
||||
explicit=[_r(True, score=0.9, weight=2.0), _r(False, score=0.3, weight=1.0)],
|
||||
rule_logic=RuleLogic.WEIGHTED,
|
||||
threshold=0.6,
|
||||
)
|
||||
# (0.9*2 + 0.3*1) / 3 = 0.7 >= 0.6
|
||||
assert outcome.passed is True
|
||||
|
||||
|
||||
def test_weighted_logic_below_threshold_fails():
|
||||
outcome = combine_case_outcome(
|
||||
all_replied=True,
|
||||
explicit=[_r(True, score=0.9, weight=1.0), _r(False, score=0.1, weight=2.0)],
|
||||
rule_logic=RuleLogic.WEIGHTED,
|
||||
threshold=0.6,
|
||||
)
|
||||
# (0.9 + 0.1*2) / 3 ≈ 0.367 < 0.6
|
||||
assert outcome.passed is False
|
||||
|
||||
|
||||
def test_weighted_logic_scoreless_rule_uses_binary_score():
|
||||
# 无 score 的规则按通过=1.0 / 失败=0.0 计(与引擎既有口径一致)
|
||||
outcome = combine_case_outcome(
|
||||
all_replied=True,
|
||||
explicit=[_r(True, score=None, weight=1.0)],
|
||||
rule_logic=RuleLogic.WEIGHTED,
|
||||
threshold=0.6,
|
||||
)
|
||||
assert outcome.passed is True
|
||||
|
||||
|
||||
# ── 期望叠加(隐式规则是硬约束) ───────────────────────────────────
|
||||
|
||||
|
||||
def test_implicit_failure_vetoes_explicit_pass():
|
||||
outcome = combine_case_outcome(
|
||||
all_replied=True, explicit=[_r(True)], implicit=[_r(False)]
|
||||
)
|
||||
assert outcome.passed is False
|
||||
|
||||
|
||||
def test_implicit_failure_vetoes_even_any_logic():
|
||||
# 隐式规则不参与 rule_logic 组合:any 满足也救不回隐式失败
|
||||
outcome = combine_case_outcome(
|
||||
all_replied=True,
|
||||
explicit=[_r(True), _r(False)],
|
||||
implicit=[_r(False)],
|
||||
rule_logic=RuleLogic.ANY,
|
||||
)
|
||||
assert outcome.passed is False
|
||||
|
||||
|
||||
def test_implicit_only_case_passes_when_implicit_pass():
|
||||
# 仅期望无显式规则:显式组合空真
|
||||
outcome = combine_case_outcome(all_replied=True, implicit=[_r(True)])
|
||||
assert outcome == CaseOutcome(passed=True, connectivity=False)
|
||||
|
||||
|
||||
def test_explicit_failure_not_saved_by_implicit_pass():
|
||||
outcome = combine_case_outcome(
|
||||
all_replied=True, explicit=[_r(False)], implicit=[_r(True)]
|
||||
)
|
||||
assert outcome.passed is False
|
||||
|
||||
|
||||
# ── 有规则用例的回复缺失:判定权在规则 ─────────────────────────────
|
||||
|
||||
|
||||
def test_ruled_case_judged_by_rules_even_with_missing_reply():
|
||||
# 有判定标准时由规则说了算(规则自身会因无回复而失败),不因缺回复直接判死
|
||||
outcome = combine_case_outcome(all_replied=False, explicit=[_r(True)])
|
||||
assert outcome == CaseOutcome(passed=True, connectivity=False)
|
||||
@ -36,3 +36,21 @@ def test_mark_orphans_failed_noop_when_clean(db_session):
|
||||
repo = RunRepository(db_session)
|
||||
_make_run(db_session, RunStatus.COMPLETED)
|
||||
assert repo.mark_orphans_failed() == 0
|
||||
|
||||
|
||||
def test_update_preserves_scenario_version_and_triggered_by(db_session):
|
||||
"""update() 不得丢字段:scenario_version / triggered_by 必须回写(漂移回归)。"""
|
||||
from agenteval.models import RunTrigger
|
||||
|
||||
repo = RunRepository(db_session)
|
||||
run = repo.create(EvalRun(
|
||||
target_id="t-1", scenario_id="s-1", status=RunStatus.RUNNING,
|
||||
scenario_version=4, triggered_by=RunTrigger.AI_ASSISTANT,
|
||||
))
|
||||
run.scenario_version = 5
|
||||
run.triggered_by = RunTrigger.CLI
|
||||
run.status = RunStatus.COMPLETED
|
||||
updated = repo.update(run)
|
||||
|
||||
assert updated.scenario_version == 5
|
||||
assert updated.triggered_by == RunTrigger.CLI
|
||||
|
||||
@ -361,3 +361,52 @@ def test_render_json_report_is_valid_json(report_session):
|
||||
json_text = render_json_report(run_id, report_session)
|
||||
parsed = json.loads(json_text)
|
||||
assert parsed["run_id"] == run_id
|
||||
|
||||
|
||||
# ── 权威判定消费(judgement 语义收敛) ────────────────────────────────────
|
||||
|
||||
def test_case_dict_contains_passed_fallback(report_session):
|
||||
"""老 run(无 case_outcomes):passed 反推得出,故障用例=False(ADR-0002)。"""
|
||||
run_id = _seed_run(report_session, pass_rate=0.5, n_cases=2,
|
||||
connectivity_cases=1, errored_cases=1)
|
||||
report = generate_report(run_id, report_session)
|
||||
by_id = {c["case_id"]: c for c in report["cases"]}
|
||||
assert by_id["c0"]["passed"] is True
|
||||
assert by_id["c1"]["passed"] is False
|
||||
assert by_id["conn0"]["passed"] is True
|
||||
assert by_id["err0"]["passed"] is False
|
||||
|
||||
|
||||
def test_markdown_errored_case_shows_failed_badge(report_session):
|
||||
"""故障用例(无结果且非连通)在 MD 中必须 ❌ —— 此前 all([]) 误判 ✅。"""
|
||||
run_id = _seed_run(report_session, n_cases=1, errored_cases=1)
|
||||
md = render_markdown_report(run_id, report_session)
|
||||
assert "❌ 用例 `err0`" in md
|
||||
assert "✅ 用例 `err0`" not in md
|
||||
|
||||
|
||||
def test_authoritative_case_outcomes_override_reconstruction(report_session):
|
||||
"""summary.case_outcomes 是权威判定:与规则结果反推冲突时以权威为准。"""
|
||||
run_id = _seed_run(report_session, pass_rate=1.0, n_cases=1)
|
||||
repo = RunRepository(report_session)
|
||||
run = repo.get(run_id)
|
||||
# 模拟 weighted 阈值未达:规则单条通过但用例判失败(反推 all() 会误判 True)
|
||||
run.summary = {**run.summary, "case_outcomes": {"c0": {"passed": False, "connectivity": False}}}
|
||||
repo.update(run)
|
||||
|
||||
report = generate_report(run_id, report_session)
|
||||
assert report["cases"][0]["passed"] is False
|
||||
md = render_markdown_report(run_id, report_session)
|
||||
assert "❌ 用例 `c0`" in md
|
||||
|
||||
|
||||
def test_compare_errored_case_counts_as_failed(report_session):
|
||||
"""对比中故障用例判 False 而非 None(ADR-0002 服务视角)。"""
|
||||
run_id_a = _seed_run(report_session, n_cases=1, errored_cases=1)
|
||||
sid = _scenario_of(report_session, run_id_a)
|
||||
run_id_b = _seed_run(report_session, n_cases=1, errored_cases=1, scenario_id=sid)
|
||||
result = generate_compare_report(run_id_a, run_id_b, report_session)
|
||||
by_id = {c["case_id"]: c for c in result["cases"]}
|
||||
assert by_id["err0"]["run_a_passed"] is False
|
||||
assert by_id["err0"]["run_b_passed"] is False
|
||||
assert by_id["err0"]["changed"] is False
|
||||
|
||||
Loading…
Reference in New Issue
Block a user