From 8a526599ab7053389b022361142085da3c0c4c55 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Wed, 29 Jul 2026 10:32:56 +0800 Subject: [PATCH] feat(report): annotate connectivity cases and add judged pass rate (ticket 02) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 报告层推导连通用例标记（无判定结果 + 每轮有回复 + 无用例级错误）， summary 新增 connectivity_cases 与 judged_pass_rate（无判定型用例时为 null）。 对比报告同步标注且连通用例按引擎口径计通过；总通过率口径不变（ADR-0002）。 --- .../issues/02-connectivity-case-annotation.md | 12 +- backend/agenteval/evaluation/report.py | 51 +++++++-- frontend/web/src/pages/Reports.tsx | 23 +++- tests/unit/test_report.py | 105 +++++++++++++++++- 4 files changed, 167 insertions(+), 24 deletions(-) diff --git a/.scratch/v0.5/issues/02-connectivity-case-annotation.md b/.scratch/v0.5/issues/02-connectivity-case-annotation.md index 04c4adb..f91c821 100644 --- a/.scratch/v0.5/issues/02-connectivity-case-annotation.md +++ b/.scratch/v0.5/issues/02-connectivity-case-annotation.md @@ -4,10 +4,10 @@ **Blocked by:** None — can start immediately. -**Status:** ready-for-agent +**Status:** done -- [ ] 单次报告中连通用例带明确标记，前端可见 -- [ ] 对比报告中连通用例同样标注 -- [ ] summary 含连通用例数与判定型通过率；全为连通用例时判定型通过率不除零 -- [ ] 总通过率数值与升级前一致（口径未变） -- [ ] 报告单元测试覆盖标注与判定型通过率计算（先例：现有报告测试） +- [x] 单次报告中连通用例带明确标记，前端可见 +- [x] 对比报告中连通用例同样标注 +- [x] summary 含连通用例数与判定型通过率；全为连通用例时判定型通过率不除零 +- [x] 总通过率数值与升级前一致（口径未变） +- [x] 报告单元测试覆盖标注与判定型通过率计算（先例：现有报告测试） diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index 68a7bcd..2d58e6e 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -111,9 +111,11 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: # Group by case case_map: dict[str, dict[str, Any]] = {} for turn in turns: - case_map.setdefault(turn.case_id, {"turns": [], "results": []}) + case_map.setdefault(turn.case_id, {"turns": [], "results": [], "all_replied": True}) sent = turn.get_sent_message() reply = turn.get_reply() + if reply is None: + case_map[turn.case_id]["all_replied"] = False case_map[turn.case_id]["turns"].append( { "round": turn.round_index, @@ -125,7 +127,7 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: ) for result in results: - case_map.setdefault(result.case_id, {"turns": [], "results": []}) + case_map.setdefault(result.case_id, {"turns": [], "results": [], "all_replied": True}) case_map[result.case_id]["results"].append( { "rule_type": result.rule_type, @@ -135,18 +137,35 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: } ) + summary = run.summary or {} + errored_case_ids = {e.get("case_id") for e in summary.get("case_errors", [])} + cases = [] for case_id in sorted(case_map.keys()): item = case_map[case_id] + # 连通用例：无任何判定结果，且每轮都收到回复、无用例级错误（CONTEXT.md） + connectivity = ( + not item["results"] + and bool(item["turns"]) + and item["all_replied"] + and case_id not in errored_case_ids + ) cases.append( { "case_id": case_id, + "connectivity": connectivity, "turns": sorted(item["turns"], key=lambda x: x["round"]), "results": item["results"], } ) - summary = run.summary or {} + total_cases = summary.get("total_cases", 0) + passed_cases = summary.get("passed_cases", 0) + connectivity_count = sum(1 for c in cases if c["connectivity"]) + judged_total = total_cases - connectivity_count + # 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数 + judged_pass_rate = round((passed_cases - connectivity_count) / judged_total, 4) if judged_total > 0 else None + return { "run_id": run.id, "target_id": run.target_id, @@ -157,12 +176,14 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: "started_at": iso_utc(run.started_at), "completed_at": iso_utc(run.completed_at), "summary": { - "total_cases": summary.get("total_cases", 0), - "passed_cases": summary.get("passed_cases", 0), + "total_cases": total_cases, + "passed_cases": passed_cases, "failed_cases": summary.get("failed_cases", 0), "total_rules": summary.get("total_rules", 0), "passed_rules": summary.get("passed_rules", 0), "pass_rate": summary.get("pass_rate", 0.0), + "connectivity_cases": connectivity_count, + "judged_pass_rate": judged_pass_rate, }, "cases": cases, } @@ -194,6 +215,9 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[ def _case_passed(c): if not c: return None + if c.get("connectivity"): + # 连通用例收到回复即通过（引擎口径） + return True results = c.get("results", []) if not results: # No rule results (e.g. errored case) must not count as passed. @@ -203,6 +227,7 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[ case_diffs.append( { "case_id": cid, + "connectivity": bool((ca and ca.get("connectivity")) or (cb and cb.get("connectivity"))), "run_a_passed": _case_passed(ca), "run_b_passed": _case_passed(cb), "changed": _case_passed(ca) != _case_passed(cb), @@ -242,6 +267,8 @@ def render_markdown_report(run_id: str, session=None) -> str: """Render a report as Markdown string.""" report = generate_report(run_id, session) s = report["summary"] + judged_rate = s.get("judged_pass_rate") + judged_rate_text = "—" if judged_rate is None else f"{judged_rate * 100:.1f}%" lines: list[str] = [ f"# 评测报告 — {report.get('scenario_name', run_id)}", "", @@ -261,15 +288,23 @@ def render_markdown_report(run_id: str, session=None) -> str: f"| 总规则数 | {s['total_rules']} |", f"| 通过规则 | {s['passed_rules']} |", f"| 通过率 | {s['pass_rate'] * 100:.1f}% |", + f"| 连通用例 | {s.get('connectivity_cases', 0)} |", + f"| 判定型通过率 | {judged_rate_text} |", "", "## 用例明细", "", ] for case in report.get("cases", []): - case_passed = all(r["passed"] for r in case.get("results", [])) - badge = "✅" if case_passed else "❌" - lines.append(f"### {badge} 用例 `{case['case_id']}`") + if case.get("connectivity"): + badge = "🔗" + else: + case_passed = all(r["passed"] for r in case.get("results", [])) + badge = "✅" if case_passed else "❌" + title = f"### {badge} 用例 `{case['case_id']}`" + if case.get("connectivity"): + title += "（连通用例，未配置判定标准）" + lines.append(title) lines.append("") for turn in case.get("turns", []): diff --git a/frontend/web/src/pages/Reports.tsx b/frontend/web/src/pages/Reports.tsx index 2e5f963..2f98532 100644 --- a/frontend/web/src/pages/Reports.tsx +++ b/frontend/web/src/pages/Reports.tsx @@ -1,7 +1,7 @@ import { useEffect, useMemo, useState } from 'react' import { useSearchParams } from 'react-router-dom' import { - Button, Card, Col, Collapse, Descriptions, Empty, Row, Segmented, + Alert, Button, Card, Col, Collapse, Descriptions, Empty, Row, Segmented, Select, Space, Spin, Statistic, Table, Tag, Tooltip, Badge, message, } from 'antd' import { @@ -29,6 +29,7 @@ interface RuleResultData { interface CaseReport { case_id: string + connectivity: boolean turns: TurnData[] results: RuleResultData[] } @@ -47,6 +48,8 @@ interface Report { total_rules: number passed_rules: number pass_rate: number + connectivity_cases: number + judged_pass_rate: number | null } cases: CaseReport[] } @@ -57,6 +60,7 @@ interface CompareResult { delta: { pass_rate: number; passed_cases: number; passed_rules: number } cases: Array<{ case_id: string + connectivity: boolean run_a_passed: boolean | null run_b_passed: boolean | null changed: boolean @@ -364,6 +368,14 @@ function SingleReportView({ report }: { report: Report | null }) { + {report.summary.connectivity_cases > 0 && ( + + )} + {report.target_name} @@ -379,9 +391,11 @@ function SingleReportView({ report }: { report: Report | null }) { label: ( {c.case_id} - {c.results.every((r) => r.passed) - ? 全部通过 - : 存在失败} + {c.connectivity + ? 连通用例 + : c.results.every((r) => r.passed) + ? 全部通过 + : 存在失败} {c.turns.length} 轮对话 ), @@ -501,6 +515,7 @@ function CompareView({ result }: { result: CompareResult | null }) { {r.changed && } {id} + {r.connectivity && 连通} ), }, diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py index 91d9e9b..c9c4d05 100644 --- a/tests/unit/test_report.py +++ b/tests/unit/test_report.py @@ -40,8 +40,14 @@ def _seed_run( pass_rate: float = 1.0, n_cases: int = 1, scenario_id: str | None = None, + connectivity_cases: int = 0, + errored_cases: int = 0, ) -> str: - """Create a minimal completed run with real data in the DB and return run_id.""" + """Create a minimal completed run with real data in the DB and return run_id. + + connectivity_cases: extra cases with replied turns but no rule results. + errored_cases: extra cases with a missing-reply turn and no rule results. + """ target = EvalTarget( name="测试对象", platform=PlatformType.AI_DIGITAL_EMPLOYEE, @@ -54,7 +60,11 @@ def _seed_run( if scenario_id is None: scenario = Scenario( name="测试场景", - cases=[Case(id=f"c{i}", type=CaseType.SINGLE, messages=["hi"]) for i in range(n_cases)], + cases=( + [Case(id=f"c{i}", type=CaseType.SINGLE, messages=["hi"]) for i in range(n_cases)] + + [Case(id=f"conn{i}", type=CaseType.SINGLE, messages=["ping"]) for i in range(connectivity_cases)] + + [Case(id=f"err{i}", type=CaseType.SINGLE, messages=["ping"]) for i in range(errored_cases)] + ), ) scenario = ScenarioRepository(session).create(scenario) scenario_id = scenario.id @@ -95,13 +105,35 @@ def _seed_run( ) result_repo.save_result(eval_result) + for i in range(connectivity_cases): + result_repo.save_turn(Turn( + run_id=run.id, + case_id=f"conn{i}", + round_index=1, + sent_message={"msgBody": {"content": "ping"}}, + reply={"msgBody": {"content": "pong"}}, + latency_ms=100, + )) + + for i in range(errored_cases): + result_repo.save_turn(Turn( + run_id=run.id, + case_id=f"err{i}", + round_index=1, + sent_message={"msgBody": {"content": "ping"}}, + reply=None, + latency_ms=None, + )) + + all_total = total + connectivity_cases + errored_cases + all_passed = passed + connectivity_cases # 连通用例收到回复即通过（引擎口径） run.summary = { - "total_cases": total, - "passed_cases": passed, - "failed_cases": total - passed, + "total_cases": all_total, + "passed_cases": all_passed, + "failed_cases": all_total - all_passed, "total_rules": total, "passed_rules": passed, - "pass_rate": round(pass_rate, 4), + "pass_rate": round(all_passed / all_total, 4) if all_total else 0.0, } RunRepository(session).update(run) return run.id @@ -210,6 +242,67 @@ def test_compare_report_different_scenarios_rejected(report_session): generate_compare_report(run_id_a, run_id_b, report_session) +# ── connectivity case annotation (ticket 02) ───────────────────────────── + +def test_report_marks_connectivity_case(report_session): + run_id = _seed_run(report_session, pass_rate=0.5, n_cases=2, connectivity_cases=1) + report = generate_report(run_id, report_session) + + by_id = {c["case_id"]: c for c in report["cases"]} + assert by_id["conn0"]["connectivity"] is True + assert by_id["c0"]["connectivity"] is False + assert by_id["c1"]["connectivity"] is False + + s = report["summary"] + assert s["connectivity_cases"] == 1 + # 判定型通过率 = 判定型通过 1 ÷ 判定型总数 2 + assert s["judged_pass_rate"] == 0.5 + # 总通过率口径不变（含连通用例，ADR-0002） + assert s["pass_rate"] == round(2 / 3, 4) + + +def test_errored_case_not_marked_connectivity(report_session): + """无判定结果但缺回复的用例是执行失败，不是连通用例。""" + run_id = _seed_run(report_session, n_cases=1, errored_cases=1) + report = generate_report(run_id, report_session) + + by_id = {c["case_id"]: c for c in report["cases"]} + assert by_id["err0"]["connectivity"] is False + assert report["summary"]["connectivity_cases"] == 0 + + +def test_all_connectivity_judged_pass_rate_none(report_session): + """全为连通用例时判定型通过率为 None，不除零。""" + run_id = _seed_run(report_session, n_cases=0, connectivity_cases=2) + report = generate_report(run_id, report_session) + + s = report["summary"] + assert s["connectivity_cases"] == 2 + assert s["judged_pass_rate"] is None + + +def test_compare_report_marks_connectivity(report_session): + run_id_a = _seed_run(report_session, n_cases=1, connectivity_cases=1) + sid = _scenario_of(report_session, run_id_a) + run_id_b = _seed_run(report_session, n_cases=1, connectivity_cases=1, scenario_id=sid) + result = generate_compare_report(run_id_a, run_id_b, report_session) + + by_id = {c["case_id"]: c for c in result["cases"]} + assert by_id["conn0"]["connectivity"] is True + assert by_id["c0"]["connectivity"] is False + # 连通用例双方均按引擎口径视为通过，不应标记 changed + assert by_id["conn0"]["run_a_passed"] is True + assert by_id["conn0"]["run_b_passed"] is True + assert by_id["conn0"]["changed"] is False + + +def test_markdown_report_shows_connectivity(report_session): + run_id = _seed_run(report_session, n_cases=1, connectivity_cases=1) + md = render_markdown_report(run_id, report_session) + assert "连通用例" in md + assert "判定型通过率" in md + + # ── render_markdown_report ──────────────────────────────────────────────── def test_render_markdown_contains_header(report_session):