diff --git a/.scratch/v0.5/issues/02-connectivity-case-annotation.md b/.scratch/v0.5/issues/02-connectivity-case-annotation.md
index 04c4adb..f91c821 100644
--- a/.scratch/v0.5/issues/02-connectivity-case-annotation.md
+++ b/.scratch/v0.5/issues/02-connectivity-case-annotation.md
@@ -4,10 +4,10 @@
**Blocked by:** None — can start immediately.
-**Status:** ready-for-agent
+**Status:** done
-- [ ] 单次报告中连通用例带明确标记，前端可见
-- [ ] 对比报告中连通用例同样标注
-- [ ] summary 含连通用例数与判定型通过率；全为连通用例时判定型通过率不除零
-- [ ] 总通过率数值与升级前一致（口径未变）
-- [ ] 报告单元测试覆盖标注与判定型通过率计算（先例：现有报告测试）
+- [x] 单次报告中连通用例带明确标记，前端可见
+- [x] 对比报告中连通用例同样标注
+- [x] summary 含连通用例数与判定型通过率；全为连通用例时判定型通过率不除零
+- [x] 总通过率数值与升级前一致（口径未变）
+- [x] 报告单元测试覆盖标注与判定型通过率计算（先例：现有报告测试）
diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py
index 68a7bcd..2d58e6e 100644
--- a/backend/agenteval/evaluation/report.py
+++ b/backend/agenteval/evaluation/report.py
@@ -111,9 +111,11 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
# Group by case
case_map: dict[str, dict[str, Any]] = {}
for turn in turns:
- case_map.setdefault(turn.case_id, {"turns": [], "results": []})
+ case_map.setdefault(turn.case_id, {"turns": [], "results": [], "all_replied": True})
sent = turn.get_sent_message()
reply = turn.get_reply()
+ if reply is None:
+ case_map[turn.case_id]["all_replied"] = False
case_map[turn.case_id]["turns"].append(
{
"round": turn.round_index,
@@ -125,7 +127,7 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
)
for result in results:
- case_map.setdefault(result.case_id, {"turns": [], "results": []})
+ case_map.setdefault(result.case_id, {"turns": [], "results": [], "all_replied": True})
case_map[result.case_id]["results"].append(
{
"rule_type": result.rule_type,
@@ -135,18 +137,35 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
}
)
+ summary = run.summary or {}
+ errored_case_ids = {e.get("case_id") for e in summary.get("case_errors", [])}
+
cases = []
for case_id in sorted(case_map.keys()):
item = case_map[case_id]
+ # 连通用例：无任何判定结果，且每轮都收到回复、无用例级错误（CONTEXT.md）
+ connectivity = (
+ not item["results"]
+ and bool(item["turns"])
+ and item["all_replied"]
+ and case_id not in errored_case_ids
+ )
cases.append(
{
"case_id": case_id,
+ "connectivity": connectivity,
"turns": sorted(item["turns"], key=lambda x: x["round"]),
"results": item["results"],
}
)
- summary = run.summary or {}
+ total_cases = summary.get("total_cases", 0)
+ passed_cases = summary.get("passed_cases", 0)
+ connectivity_count = sum(1 for c in cases if c["connectivity"])
+ judged_total = total_cases - connectivity_count
+ # 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数
+ judged_pass_rate = round((passed_cases - connectivity_count) / judged_total, 4) if judged_total > 0 else None
+
return {
"run_id": run.id,
"target_id": run.target_id,
@@ -157,12 +176,14 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
"started_at": iso_utc(run.started_at),
"completed_at": iso_utc(run.completed_at),
"summary": {
- "total_cases": summary.get("total_cases", 0),
- "passed_cases": summary.get("passed_cases", 0),
+ "total_cases": total_cases,
+ "passed_cases": passed_cases,
"failed_cases": summary.get("failed_cases", 0),
"total_rules": summary.get("total_rules", 0),
"passed_rules": summary.get("passed_rules", 0),
"pass_rate": summary.get("pass_rate", 0.0),
+ "connectivity_cases": connectivity_count,
+ "judged_pass_rate": judged_pass_rate,
},
"cases": cases,
}
@@ -194,6 +215,9 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[
def _case_passed(c):
if not c:
return None
+ if c.get("connectivity"):
+ # 连通用例收到回复即通过（引擎口径）
+ return True
results = c.get("results", [])
if not results:
# No rule results (e.g. errored case) must not count as passed.
@@ -203,6 +227,7 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[
case_diffs.append(
{
"case_id": cid,
+ "connectivity": bool((ca and ca.get("connectivity")) or (cb and cb.get("connectivity"))),
"run_a_passed": _case_passed(ca),
"run_b_passed": _case_passed(cb),
"changed": _case_passed(ca) != _case_passed(cb),
@@ -242,6 +267,8 @@ def render_markdown_report(run_id: str, session=None) -> str:
"""Render a report as Markdown string."""
report = generate_report(run_id, session)
s = report["summary"]
+ judged_rate = s.get("judged_pass_rate")
+ judged_rate_text = "—" if judged_rate is None else f"{judged_rate * 100:.1f}%"
lines: list[str] = [
f"# 评测报告 — {report.get('scenario_name', run_id)}",
"",
@@ -261,15 +288,23 @@ def render_markdown_report(run_id: str, session=None) -> str:
f"| 总规则数 | {s['total_rules']} |",
f"| 通过规则 | {s['passed_rules']} |",
f"| 通过率 | {s['pass_rate'] * 100:.1f}% |",
+ f"| 连通用例 | {s.get('connectivity_cases', 0)} |",
+ f"| 判定型通过率 | {judged_rate_text} |",
"",
"## 用例明细",
"",
]
for case in report.get("cases", []):
- case_passed = all(r["passed"] for r in case.get("results", []))
- badge = "✅" if case_passed else "❌"
- lines.append(f"### {badge} 用例 `{case['case_id']}`")
+ if case.get("connectivity"):
+ badge = "🔗"
+ else:
+ case_passed = all(r["passed"] for r in case.get("results", []))
+ badge = "✅" if case_passed else "❌"
+ title = f"### {badge} 用例 `{case['case_id']}`"
+ if case.get("connectivity"):
+ title += "（连通用例，未配置判定标准）"
+ lines.append(title)
lines.append("")
for turn in case.get("turns", []):
diff --git a/frontend/web/src/pages/Reports.tsx b/frontend/web/src/pages/Reports.tsx
index 2e5f963..2f98532 100644
--- a/frontend/web/src/pages/Reports.tsx
+++ b/frontend/web/src/pages/Reports.tsx
@@ -1,7 +1,7 @@
import { useEffect, useMemo, useState } from 'react'
import { useSearchParams } from 'react-router-dom'
import {
- Button, Card, Col, Collapse, Descriptions, Empty, Row, Segmented,
+ Alert, Button, Card, Col, Collapse, Descriptions, Empty, Row, Segmented,
Select, Space, Spin, Statistic, Table, Tag, Tooltip, Badge, message,
} from 'antd'
import {
@@ -29,6 +29,7 @@ interface RuleResultData {
interface CaseReport {
case_id: string
+ connectivity: boolean
turns: TurnData[]
results: RuleResultData[]
}
@@ -47,6 +48,8 @@ interface Report {
total_rules: number
passed_rules: number
pass_rate: number
+ connectivity_cases: number
+ judged_pass_rate: number | null
}
cases: CaseReport[]
}
@@ -57,6 +60,7 @@ interface CompareResult {
delta: { pass_rate: number; passed_cases: number; passed_rules: number }
cases: Array<{
case_id: string
+ connectivity: boolean
run_a_passed: boolean | null
run_b_passed: boolean | null
changed: boolean
@@ -364,6 +368,14 @@ function SingleReportView({ report }: { report: Report | null }) {
+ {report.summary.connectivity_cases > 0 && (
+
+ )}
+
{report.target_name}
@@ -379,9 +391,11 @@ function SingleReportView({ report }: { report: Report | null }) {
label: (
{c.case_id}
- {c.results.every((r) => r.passed)
- ? 全部通过
- : 存在失败}
+ {c.connectivity
+ ? 连通用例
+ : c.results.every((r) => r.passed)
+ ? 全部通过
+ : 存在失败}
{c.turns.length} 轮对话
),
@@ -501,6 +515,7 @@ function CompareView({ result }: { result: CompareResult | null }) {
{r.changed && }
{id}
+ {r.connectivity && 连通}
),
},
diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py
index 91d9e9b..c9c4d05 100644
--- a/tests/unit/test_report.py
+++ b/tests/unit/test_report.py
@@ -40,8 +40,14 @@ def _seed_run(
pass_rate: float = 1.0,
n_cases: int = 1,
scenario_id: str | None = None,
+ connectivity_cases: int = 0,
+ errored_cases: int = 0,
) -> str:
- """Create a minimal completed run with real data in the DB and return run_id."""
+ """Create a minimal completed run with real data in the DB and return run_id.
+
+ connectivity_cases: extra cases with replied turns but no rule results.
+ errored_cases: extra cases with a missing-reply turn and no rule results.
+ """
target = EvalTarget(
name="测试对象",
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
@@ -54,7 +60,11 @@ def _seed_run(
if scenario_id is None:
scenario = Scenario(
name="测试场景",
- cases=[Case(id=f"c{i}", type=CaseType.SINGLE, messages=["hi"]) for i in range(n_cases)],
+ cases=(
+ [Case(id=f"c{i}", type=CaseType.SINGLE, messages=["hi"]) for i in range(n_cases)]
+ + [Case(id=f"conn{i}", type=CaseType.SINGLE, messages=["ping"]) for i in range(connectivity_cases)]
+ + [Case(id=f"err{i}", type=CaseType.SINGLE, messages=["ping"]) for i in range(errored_cases)]
+ ),
)
scenario = ScenarioRepository(session).create(scenario)
scenario_id = scenario.id
@@ -95,13 +105,35 @@ def _seed_run(
)
result_repo.save_result(eval_result)
+ for i in range(connectivity_cases):
+ result_repo.save_turn(Turn(
+ run_id=run.id,
+ case_id=f"conn{i}",
+ round_index=1,
+ sent_message={"msgBody": {"content": "ping"}},
+ reply={"msgBody": {"content": "pong"}},
+ latency_ms=100,
+ ))
+
+ for i in range(errored_cases):
+ result_repo.save_turn(Turn(
+ run_id=run.id,
+ case_id=f"err{i}",
+ round_index=1,
+ sent_message={"msgBody": {"content": "ping"}},
+ reply=None,
+ latency_ms=None,
+ ))
+
+ all_total = total + connectivity_cases + errored_cases
+ all_passed = passed + connectivity_cases # 连通用例收到回复即通过（引擎口径）
run.summary = {
- "total_cases": total,
- "passed_cases": passed,
- "failed_cases": total - passed,
+ "total_cases": all_total,
+ "passed_cases": all_passed,
+ "failed_cases": all_total - all_passed,
"total_rules": total,
"passed_rules": passed,
- "pass_rate": round(pass_rate, 4),
+ "pass_rate": round(all_passed / all_total, 4) if all_total else 0.0,
}
RunRepository(session).update(run)
return run.id
@@ -210,6 +242,67 @@ def test_compare_report_different_scenarios_rejected(report_session):
generate_compare_report(run_id_a, run_id_b, report_session)
+# ── connectivity case annotation (ticket 02) ─────────────────────────────
+
+def test_report_marks_connectivity_case(report_session):
+ run_id = _seed_run(report_session, pass_rate=0.5, n_cases=2, connectivity_cases=1)
+ report = generate_report(run_id, report_session)
+
+ by_id = {c["case_id"]: c for c in report["cases"]}
+ assert by_id["conn0"]["connectivity"] is True
+ assert by_id["c0"]["connectivity"] is False
+ assert by_id["c1"]["connectivity"] is False
+
+ s = report["summary"]
+ assert s["connectivity_cases"] == 1
+ # 判定型通过率 = 判定型通过 1 ÷ 判定型总数 2
+ assert s["judged_pass_rate"] == 0.5
+ # 总通过率口径不变（含连通用例，ADR-0002）
+ assert s["pass_rate"] == round(2 / 3, 4)
+
+
+def test_errored_case_not_marked_connectivity(report_session):
+ """无判定结果但缺回复的用例是执行失败，不是连通用例。"""
+ run_id = _seed_run(report_session, n_cases=1, errored_cases=1)
+ report = generate_report(run_id, report_session)
+
+ by_id = {c["case_id"]: c for c in report["cases"]}
+ assert by_id["err0"]["connectivity"] is False
+ assert report["summary"]["connectivity_cases"] == 0
+
+
+def test_all_connectivity_judged_pass_rate_none(report_session):
+ """全为连通用例时判定型通过率为 None，不除零。"""
+ run_id = _seed_run(report_session, n_cases=0, connectivity_cases=2)
+ report = generate_report(run_id, report_session)
+
+ s = report["summary"]
+ assert s["connectivity_cases"] == 2
+ assert s["judged_pass_rate"] is None
+
+
+def test_compare_report_marks_connectivity(report_session):
+ run_id_a = _seed_run(report_session, n_cases=1, connectivity_cases=1)
+ sid = _scenario_of(report_session, run_id_a)
+ run_id_b = _seed_run(report_session, n_cases=1, connectivity_cases=1, scenario_id=sid)
+ result = generate_compare_report(run_id_a, run_id_b, report_session)
+
+ by_id = {c["case_id"]: c for c in result["cases"]}
+ assert by_id["conn0"]["connectivity"] is True
+ assert by_id["c0"]["connectivity"] is False
+ # 连通用例双方均按引擎口径视为通过，不应标记 changed
+ assert by_id["conn0"]["run_a_passed"] is True
+ assert by_id["conn0"]["run_b_passed"] is True
+ assert by_id["conn0"]["changed"] is False
+
+
+def test_markdown_report_shows_connectivity(report_session):
+ run_id = _seed_run(report_session, n_cases=1, connectivity_cases=1)
+ md = render_markdown_report(run_id, report_session)
+ assert "连通用例" in md
+ assert "判定型通过率" in md
+
+
# ── render_markdown_report ────────────────────────────────────────────────
def test_render_markdown_contains_header(report_session):