From 5db0ede4f461764ff59d305e6560ac335d04761e Mon Sep 17 00:00:00 2001 From: sinohqb Date: Wed, 29 Jul 2026 19:45:02 +0800 Subject: [PATCH] refactor(judgement): converge case-pass decision into one deep module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 「用例是否通过」此前散落 8 处且互相矛盾：engine 权威判定焊死在持久化里 不可单测；report 聚合/compare/markdown 各自从规则结果反推，规则还不一致 （markdown 用 all([]) 把故障用例误渲染成 ✅）。 - 新增纯函数 evaluation/judgement.combine_case_outcome（RuleOutcome/ CaseOutcome），判定组合脱离通道与 DB 可单测（判定矩阵 14 例） - engine 调用它一次，逐用例权威结果写入 summary.case_outcomes（JSON， 零迁移）；report/compare/markdown 只读权威值，老 run fallback 反推 - 故障用例判 False（ADR-0002）：修正 markdown 的 ✅ bug 与 compare 的 None；顺带修 engine 连通用例无回复也算通过的 bug - pass_rate 口径改为用例级（CONTEXT.md 词条），规则级保留在 passed_rules/total_rules；CLI 对比标签同步更正 - 修 RunRepository.update 漏拷 scenario_version/triggered_by 的字段漂移 --- backend/agenteval/evaluation/engine.py | 88 +++++++-------- backend/agenteval/evaluation/judgement.py | 64 +++++++++++ backend/agenteval/evaluation/report.py | 45 ++++---- backend/agenteval/storage/repository.py | 2 + backend/cli/report.py | 2 +- tests/unit/test_engine.py | 39 +++++++ tests/unit/test_judgement.py | 129 ++++++++++++++++++++++ tests/unit/test_orphan_runs.py | 18 +++ tests/unit/test_report.py | 49 ++++++++ 9 files changed, 366 insertions(+), 70 deletions(-) create mode 100644 backend/agenteval/evaluation/judgement.py create mode 100644 tests/unit/test_judgement.py diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index fd17558..d2f1957 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -12,6 +12,7 @@ from typing import Any, Callable, Optional from agenteval.channels.base import EvalChannel from agenteval.channels.factory import ChannelFactory +from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome from agenteval.evaluation.rules import RuleResult, get_rule from agenteval.model_gateway import ModelGateway from agenteval.models import ( @@ -22,7 +23,6 @@ from agenteval.models import ( EvalTarget, ModelCapability, ModelPurpose, - RuleLogic, RunStatus, RunTrigger, Scenario, @@ -129,6 +129,7 @@ class EvalEngine: total_cases = len(self.scenario.cases) passed_cases = 0 failed_cases = 0 + case_outcomes: dict[str, dict[str, bool]] = {} for idx, case in enumerate(self.scenario.cases, start=1): self._check_cancel() @@ -142,12 +143,13 @@ class EvalEngine: }, ) async with self._case_semaphore: - case_passed, rule_pass, rule_total = await self._run_case( + outcome, rule_pass, rule_total = await self._run_case( run, case, progress_callback, ) - if case_passed: + case_outcomes[case.id] = {"passed": outcome.passed, "connectivity": outcome.connectivity} + if outcome.passed: passed_cases += 1 else: failed_cases += 1 @@ -158,7 +160,7 @@ class EvalEngine: "index": idx, "total": total_cases, "case_id": case.id, - "passed": case_passed, + "passed": outcome.passed, "rule_pass_count": rule_pass, "rule_total": rule_total, }, @@ -174,7 +176,10 @@ class EvalEngine: "failed_cases": failed_cases, "total_rules": total_rules, "passed_rules": passed_rules, - "pass_rate": round(passed_rules / total_rules, 4) if total_rules else 0.0, + # 通过率是用例级口径（CONTEXT.md）；规则级数字保留在 passed_rules/total_rules + "pass_rate": round(passed_cases / total_cases, 4) if total_cases else 0.0, + # 逐用例权威判定（judgement.py 算一次），报告/对比/渲染层只读不重算 + "case_outcomes": case_outcomes, } # Surface fatal case-level errors (e.g. dynamic generation failures) # so the report / DB record shows *why* a run produced no results. @@ -249,8 +254,9 @@ class EvalEngine: run: EvalRun, case: Case, progress_callback: Optional[ProgressCallback], - ) -> tuple[bool, int, int]: - """Run a single case; returns (all_rules_passed, passed_rules, total_rules).""" + ) -> tuple[CaseOutcome, int, int]: + """Run a single case; returns (outcome, passed_rules, total_rules).""" + failed = CaseOutcome(passed=False, connectivity=False) if case.type == CaseType.DYNAMIC: generated = await self._generate_messages(case, progress_callback) if not generated: @@ -262,7 +268,7 @@ class EvalEngine: "case_id": case.id, }, ) - return False, 0, 0 + return failed, 0, 0 case = case.model_copy(update={"messages": generated}) dialog: list[Turn] = [] @@ -302,7 +308,7 @@ class EvalEngine: "error": send_result.error, }, ) - return False, 0, 0 + return failed, 0, 0 try: reply = await self.channel.poll_reply( @@ -331,7 +337,7 @@ class EvalEngine: "error": f"poll_reply 异常: {poll_exc}", }, ) - return False, 0, 0 + return failed, 0, 0 received_at = utc_now() latency_ms = None @@ -366,7 +372,7 @@ class EvalEngine: ) if not dialog: - return False, 0, 0 + return failed, 0, 0 return await self._save_rule_results(run, case, dialog[-1], dialog, progress_callback) @@ -377,14 +383,11 @@ class EvalEngine: turn: Turn, dialog: list[Turn], progress_callback: Optional[ProgressCallback], - ) -> tuple[bool, int, int]: - """Apply rules and save results; returns (case_passed, passed_count, total_count). + ) -> tuple[CaseOutcome, int, int]: + """Apply rules and save results; returns (outcome, passed_count, total_count). - Judgement semantics (spec v0.5 / CONTEXT.md): - - Explicit rules are combined by case.rule_logic (ALL / ANY / WEIGHTED). - - Expectations always derive implicit checks, additive to explicit - rules. They are hard constraints: they never join the rule_logic - combination, and any implicit failure fails the case. + 判定组合本身在 judgement.combine_case_outcome（单一权威）—— + 本方法只负责执行规则、持久化结果并把规则输出规范化为 RuleOutcome。 """ from agenteval.models import EvalRuleConfig @@ -411,17 +414,16 @@ class EvalEngine: ) ) + all_replied = bool(dialog) and all(t.reply is not None for t in dialog) + if not rules_config and not implicit_config: - # 连通用例：无任何判定标准，收到回复即通过 - return True, 0, 0 + # 连通用例：收到全部回复才通过（无回复=故障，ADR-0002） + return combine_case_outcome(all_replied=all_replied), 0, 0 passed_count = 0 total_count = 0 - explicit_passed = 0 - explicit_total = 0 - weighted_score = 0.0 - total_weight = 0.0 - implicit_all_passed = True + explicit_outcomes: list[RuleOutcome] = [] + implicit_outcomes: list[RuleOutcome] = [] all_rules = [(cfg, False) for cfg in rules_config] + [(cfg, True) for cfg in implicit_config] for rule_config, is_implicit in all_rules: @@ -457,18 +459,11 @@ class EvalEngine: if result.passed: passed_count += 1 + rule_outcome = RuleOutcome(passed=result.passed, score=result.score, weight=rule_config.weight) if is_implicit: - if not result.passed: - implicit_all_passed = False + implicit_outcomes.append(rule_outcome) else: - explicit_total += 1 - if result.passed: - explicit_passed += 1 - # Weighted scoring: use rule score (default 1.0 if passed, 0.0 if failed) - score_val = result.score if result.score is not None else (1.0 if result.passed else 0.0) - weight = rule_config.weight - weighted_score += score_val * weight - total_weight += weight + explicit_outcomes.append(rule_outcome) await self._emit( progress_callback, @@ -484,21 +479,14 @@ class EvalEngine: }, ) - # Combine explicit rules by rule_logic; no explicit rules → vacuously true. - logic = case.rule_logic - if not rules_config: - explicit_ok = True - elif logic == RuleLogic.ANY: - explicit_ok = explicit_passed > 0 - elif logic == RuleLogic.WEIGHTED: - avg = weighted_score / total_weight if total_weight > 0 else 0.0 - explicit_ok = avg >= case.rule_pass_threshold - else: # ALL and fallback - explicit_ok = explicit_passed == explicit_total - - case_passed = explicit_ok and implicit_all_passed - - return case_passed, passed_count, total_count + outcome = combine_case_outcome( + all_replied=all_replied, + explicit=explicit_outcomes, + implicit=implicit_outcomes, + rule_logic=case.rule_logic, + threshold=case.rule_pass_threshold, + ) + return outcome, passed_count, total_count async def _generate_messages( self, diff --git a/backend/agenteval/evaluation/judgement.py b/backend/agenteval/evaluation/judgement.py new file mode 100644 index 0000000..a83e6cc --- /dev/null +++ b/backend/agenteval/evaluation/judgement.py @@ -0,0 +1,64 @@ +"""Case judgement — the single authority for "did this case pass". + +判定语义（CONTEXT.md / spec v0.5 / ADR-0002）： + - 连通用例（无显式规则、无期望）：每轮收到回复即通过；缺回复=故障=不通过。 + - 显式规则按 rule_logic（ALL / ANY / WEIGHTED）组合；无显式规则时空真。 + - 期望派生的隐式规则是 rule_logic 之外的硬约束，任一失败则用例失败。 + +引擎执行时调用一次，结果写入 run.summary["case_outcomes"]， +报告 / 对比 / 渲染层只消费该权威值，不得各自重算。 +""" + +from dataclasses import dataclass +from typing import Optional, Sequence + +from agenteval.models import RuleLogic + + +@dataclass(frozen=True) +class RuleOutcome: + """单条规则的判定结果（引擎执行规则后规范化为此形状）。""" + + passed: bool + score: Optional[float] = None + weight: float = 1.0 + + +@dataclass(frozen=True) +class CaseOutcome: + """单个用例的权威判定结果。""" + + passed: bool + connectivity: bool + + +def combine_case_outcome( + *, + all_replied: bool, + explicit: Sequence[RuleOutcome] = (), + implicit: Sequence[RuleOutcome] = (), + rule_logic: RuleLogic = RuleLogic.ALL, + threshold: float = 0.6, +) -> CaseOutcome: + if not explicit and not implicit: + # 连通用例：connectivity 标记仅在连通成功时为真 + ok = all_replied + return CaseOutcome(passed=ok, connectivity=ok) + + if not explicit: + explicit_ok = True + elif rule_logic == RuleLogic.ANY: + explicit_ok = any(r.passed for r in explicit) + elif rule_logic == RuleLogic.WEIGHTED: + total_weight = sum(r.weight for r in explicit) + weighted_score = sum( + (r.score if r.score is not None else (1.0 if r.passed else 0.0)) * r.weight + for r in explicit + ) + avg = weighted_score / total_weight if total_weight > 0 else 0.0 + explicit_ok = avg >= threshold + else: # ALL and fallback + explicit_ok = all(r.passed for r in explicit) + + implicit_ok = all(r.passed for r in implicit) + return CaseOutcome(passed=explicit_ok and implicit_ok, connectivity=False) diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index 1351285..e7225e0 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -139,20 +139,36 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: summary = run.summary or {} errored_case_ids = {e.get("case_id") for e in summary.get("case_errors", [])} + # 权威判定：引擎经 judgement.combine_case_outcome 算一次写入 summary； + # 老 run 没有该字段时退回从持久化结果反推（WEIGHTED/ANY 只能近似）。 + authoritative: dict[str, Any] = summary.get("case_outcomes", {}) cases = [] for case_id in sorted(case_map.keys()): item = case_map[case_id] - # 连通用例：无任何判定结果，且每轮都收到回复、无用例级错误（CONTEXT.md） - connectivity = ( - not item["results"] - and bool(item["turns"]) - and item["all_replied"] - and case_id not in errored_case_ids - ) + if case_id in authoritative: + outcome = authoritative[case_id] + connectivity = bool(outcome.get("connectivity")) + passed = bool(outcome.get("passed")) + else: + # 连通用例：无任何判定结果，且每轮都收到回复、无用例级错误（CONTEXT.md） + connectivity = ( + not item["results"] + and bool(item["turns"]) + and item["all_replied"] + and case_id not in errored_case_ids + ) + if connectivity: + passed = True + elif not item["results"]: + # 故障用例（无结果且非连通）＝不通过（ADR-0002） + passed = False + else: + passed = all(r["passed"] for r in item["results"]) cases.append( { "case_id": case_id, + "passed": passed, "connectivity": connectivity, "turns": sorted(item["turns"], key=lambda x: x["round"]), "results": item["results"], @@ -221,16 +237,8 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[ cb = cases_b.get(cid) def _case_passed(c): - if not c: - return None - if c.get("connectivity"): - # 连通用例收到回复即通过（引擎口径） - return True - results = c.get("results", []) - if not results: - # No rule results (e.g. errored case) must not count as passed. - return None - return all(r["passed"] for r in results) + # None 仅表示该 run 没有这个用例；判定本身读权威 passed 字段 + return c["passed"] if c else None case_diffs.append( { @@ -311,8 +319,7 @@ def render_markdown_report(run_id: str, session=None) -> str: if case.get("connectivity"): badge = "🔗" else: - case_passed = all(r["passed"] for r in case.get("results", [])) - badge = "✅" if case_passed else "❌" + badge = "✅" if case.get("passed") else "❌" title = f"### {badge} 用例 `{case['case_id']}`" if case.get("connectivity"): title += "（连通用例，未配置判定标准）" diff --git a/backend/agenteval/storage/repository.py b/backend/agenteval/storage/repository.py index 8ffddc9..024db12 100644 --- a/backend/agenteval/storage/repository.py +++ b/backend/agenteval/storage/repository.py @@ -302,7 +302,9 @@ class RunRepository: return None existing.target_id = run.target_id existing.scenario_id = run.scenario_id + existing.scenario_version = run.scenario_version existing.status = run.status.value + existing.triggered_by = run.triggered_by.value existing.completed_at = run.completed_at if run.summary: existing.set_summary(run.summary) diff --git a/backend/cli/report.py b/backend/cli/report.py index 894182a..f4c2eb8 100644 --- a/backend/cli/report.py +++ b/backend/cli/report.py @@ -65,4 +65,4 @@ def compare_reports( console.print(f"对比: {run_id_1} vs {run_id_2}") console.print(f" 用例数: {report1['summary']['total_cases']} -> {report2['summary']['total_cases']}") console.print(f" 通过用例: {report1['summary']['passed_cases']} -> {report2['summary']['passed_cases']}") - console.print(f" 规则通过率: {report1['summary']['pass_rate']:.2%} -> {report2['summary']['pass_rate']:.2%}") + console.print(f" 通过率: {report1['summary']['pass_rate']:.2%} -> {report2['summary']['pass_rate']:.2%}") diff --git a/tests/unit/test_engine.py b/tests/unit/test_engine.py index 241d040..ecb8c1a 100644 --- a/tests/unit/test_engine.py +++ b/tests/unit/test_engine.py @@ -455,3 +455,42 @@ async def test_engine_run_snapshots_scenario_version(db_session): persisted = RunRepository(db_session).get(run.id) assert persisted.scenario_version == 3 + + +# ── 权威判定写入 summary.case_outcomes（判定语义收敛） ──────────────────── + +async def test_summary_contains_case_outcomes_and_case_level_pass_rate(db_session): + scenario = Scenario(id="s-1", name="outcomes", cases=[ + # 连通用例（无规则无期望） + Case(id="conn", type=CaseType.SINGLE, messages=["ping"]), + # 判定失败用例（期望不满足） + Case(id="bad", type=CaseType.SINGLE, messages=["hi"], + expectations=Expectation(keywords_include=["__NOPE__"])), + ]) + channel = MockChannel() + engine = _build_engine(scenario, channel, session=db_session) + + run = await engine.run() + + outcomes = run.summary["case_outcomes"] + assert outcomes["conn"] == {"passed": True, "connectivity": True} + assert outcomes["bad"] == {"passed": False, "connectivity": False} + # 通过率为用例级口径（CONTEXT.md），不再是规则级 + assert run.summary["pass_rate"] == 0.5 + + +async def test_connectivity_case_without_reply_fails(db_session): + """连通用例没收到回复=故障=不通过（此前无条件判通过的 bug）。""" + scenario = Scenario(id="s-1", name="conn-fail", cases=[ + Case(id="conn", type=CaseType.SINGLE, messages=["ping"]), + ]) + channel = MockChannel(missing_reply=True) + engine = _build_engine( + scenario, channel, session=db_session, + timeout_config=TimeoutConfig(poll_reply=0.2), + ) + + run = await engine.run() + + assert run.summary["passed_cases"] == 0 + assert run.summary["case_outcomes"]["conn"] == {"passed": False, "connectivity": False} diff --git a/tests/unit/test_judgement.py b/tests/unit/test_judgement.py new file mode 100644 index 0000000..efc995e --- /dev/null +++ b/tests/unit/test_judgement.py @@ -0,0 +1,129 @@ +"""判定组合器（judgement）单元测试 — 纯函数，无通道无 DB。 + +语义来源：CONTEXT.md（连通用例/通过率）、ADR-0002（故障=不通过）、 +spec v0.5（期望叠加：隐式规则是 rule_logic 之外的硬约束）。 +""" + +from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome +from agenteval.models import RuleLogic + + +def _r(passed: bool, score: float | None = None, weight: float = 1.0) -> RuleOutcome: + return RuleOutcome(passed=passed, score=score, weight=weight) + + +# ── 连通用例（无任何判定标准） ────────────────────────────────────── + + +def test_connectivity_case_all_replied_passes(): + outcome = combine_case_outcome(all_replied=True) + assert outcome == CaseOutcome(passed=True, connectivity=True) + + +def test_connectivity_case_missing_reply_fails(): + # 无回复=故障=不通过（ADR-0002 服务视角），且不再标注为连通 + outcome = combine_case_outcome(all_replied=False) + assert outcome == CaseOutcome(passed=False, connectivity=False) + + +# ── 显式规则组合（rule_logic） ───────────────────────────────────── + + +def test_all_logic_every_rule_passes(): + outcome = combine_case_outcome(all_replied=True, explicit=[_r(True), _r(True)]) + assert outcome == CaseOutcome(passed=True, connectivity=False) + + +def test_all_logic_single_failure_fails(): + outcome = combine_case_outcome(all_replied=True, explicit=[_r(True), _r(False)]) + assert outcome.passed is False + + +def test_any_logic_single_pass_suffices(): + outcome = combine_case_outcome( + all_replied=True, explicit=[_r(False), _r(True)], rule_logic=RuleLogic.ANY + ) + assert outcome.passed is True + + +def test_any_logic_no_pass_fails(): + outcome = combine_case_outcome( + all_replied=True, explicit=[_r(False), _r(False)], rule_logic=RuleLogic.ANY + ) + assert outcome.passed is False + + +def test_weighted_logic_above_threshold_passes(): + outcome = combine_case_outcome( + all_replied=True, + explicit=[_r(True, score=0.9, weight=2.0), _r(False, score=0.3, weight=1.0)], + rule_logic=RuleLogic.WEIGHTED, + threshold=0.6, + ) + # (0.9*2 + 0.3*1) / 3 = 0.7 >= 0.6 + assert outcome.passed is True + + +def test_weighted_logic_below_threshold_fails(): + outcome = combine_case_outcome( + all_replied=True, + explicit=[_r(True, score=0.9, weight=1.0), _r(False, score=0.1, weight=2.0)], + rule_logic=RuleLogic.WEIGHTED, + threshold=0.6, + ) + # (0.9 + 0.1*2) / 3 ≈ 0.367 < 0.6 + assert outcome.passed is False + + +def test_weighted_logic_scoreless_rule_uses_binary_score(): + # 无 score 的规则按通过=1.0 / 失败=0.0 计（与引擎既有口径一致） + outcome = combine_case_outcome( + all_replied=True, + explicit=[_r(True, score=None, weight=1.0)], + rule_logic=RuleLogic.WEIGHTED, + threshold=0.6, + ) + assert outcome.passed is True + + +# ── 期望叠加（隐式规则是硬约束） ─────────────────────────────────── + + +def test_implicit_failure_vetoes_explicit_pass(): + outcome = combine_case_outcome( + all_replied=True, explicit=[_r(True)], implicit=[_r(False)] + ) + assert outcome.passed is False + + +def test_implicit_failure_vetoes_even_any_logic(): + # 隐式规则不参与 rule_logic 组合：any 满足也救不回隐式失败 + outcome = combine_case_outcome( + all_replied=True, + explicit=[_r(True), _r(False)], + implicit=[_r(False)], + rule_logic=RuleLogic.ANY, + ) + assert outcome.passed is False + + +def test_implicit_only_case_passes_when_implicit_pass(): + # 仅期望无显式规则：显式组合空真 + outcome = combine_case_outcome(all_replied=True, implicit=[_r(True)]) + assert outcome == CaseOutcome(passed=True, connectivity=False) + + +def test_explicit_failure_not_saved_by_implicit_pass(): + outcome = combine_case_outcome( + all_replied=True, explicit=[_r(False)], implicit=[_r(True)] + ) + assert outcome.passed is False + + +# ── 有规则用例的回复缺失：判定权在规则 ───────────────────────────── + + +def test_ruled_case_judged_by_rules_even_with_missing_reply(): + # 有判定标准时由规则说了算（规则自身会因无回复而失败），不因缺回复直接判死 + outcome = combine_case_outcome(all_replied=False, explicit=[_r(True)]) + assert outcome == CaseOutcome(passed=True, connectivity=False) diff --git a/tests/unit/test_orphan_runs.py b/tests/unit/test_orphan_runs.py index a665fae..8e538c9 100644 --- a/tests/unit/test_orphan_runs.py +++ b/tests/unit/test_orphan_runs.py @@ -36,3 +36,21 @@ def test_mark_orphans_failed_noop_when_clean(db_session): repo = RunRepository(db_session) _make_run(db_session, RunStatus.COMPLETED) assert repo.mark_orphans_failed() == 0 + + +def test_update_preserves_scenario_version_and_triggered_by(db_session): + """update() 不得丢字段：scenario_version / triggered_by 必须回写（漂移回归）。""" + from agenteval.models import RunTrigger + + repo = RunRepository(db_session) + run = repo.create(EvalRun( + target_id="t-1", scenario_id="s-1", status=RunStatus.RUNNING, + scenario_version=4, triggered_by=RunTrigger.AI_ASSISTANT, + )) + run.scenario_version = 5 + run.triggered_by = RunTrigger.CLI + run.status = RunStatus.COMPLETED + updated = repo.update(run) + + assert updated.scenario_version == 5 + assert updated.triggered_by == RunTrigger.CLI diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py index 011eebe..1b23918 100644 --- a/tests/unit/test_report.py +++ b/tests/unit/test_report.py @@ -361,3 +361,52 @@ def test_render_json_report_is_valid_json(report_session): json_text = render_json_report(run_id, report_session) parsed = json.loads(json_text) assert parsed["run_id"] == run_id + + +# ── 权威判定消费（judgement 语义收敛） ──────────────────────────────────── + +def test_case_dict_contains_passed_fallback(report_session): + """老 run（无 case_outcomes）：passed 反推得出，故障用例=False（ADR-0002）。""" + run_id = _seed_run(report_session, pass_rate=0.5, n_cases=2, + connectivity_cases=1, errored_cases=1) + report = generate_report(run_id, report_session) + by_id = {c["case_id"]: c for c in report["cases"]} + assert by_id["c0"]["passed"] is True + assert by_id["c1"]["passed"] is False + assert by_id["conn0"]["passed"] is True + assert by_id["err0"]["passed"] is False + + +def test_markdown_errored_case_shows_failed_badge(report_session): + """故障用例（无结果且非连通）在 MD 中必须 ❌ —— 此前 all([]) 误判 ✅。""" + run_id = _seed_run(report_session, n_cases=1, errored_cases=1) + md = render_markdown_report(run_id, report_session) + assert "❌ 用例 `err0`" in md + assert "✅ 用例 `err0`" not in md + + +def test_authoritative_case_outcomes_override_reconstruction(report_session): + """summary.case_outcomes 是权威判定：与规则结果反推冲突时以权威为准。""" + run_id = _seed_run(report_session, pass_rate=1.0, n_cases=1) + repo = RunRepository(report_session) + run = repo.get(run_id) + # 模拟 weighted 阈值未达：规则单条通过但用例判失败（反推 all() 会误判 True） + run.summary = {**run.summary, "case_outcomes": {"c0": {"passed": False, "connectivity": False}}} + repo.update(run) + + report = generate_report(run_id, report_session) + assert report["cases"][0]["passed"] is False + md = render_markdown_report(run_id, report_session) + assert "❌ 用例 `c0`" in md + + +def test_compare_errored_case_counts_as_failed(report_session): + """对比中故障用例判 False 而非 None（ADR-0002 服务视角）。""" + run_id_a = _seed_run(report_session, n_cases=1, errored_cases=1) + sid = _scenario_of(report_session, run_id_a) + run_id_b = _seed_run(report_session, n_cases=1, errored_cases=1, scenario_id=sid) + result = generate_compare_report(run_id_a, run_id_b, report_session) + by_id = {c["case_id"]: c for c in result["cases"]} + assert by_id["err0"]["run_a_passed"] is False + assert by_id["err0"]["run_b_passed"] is False + assert by_id["err0"]["changed"] is False