solahqb/AgentEvalTool
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(report): annotate connectivity cases and add judged pass rate (ticket 02)

Browse Source 
报告层推导连通用例标记（无判定结果 + 每轮有回复 + 无用例级错误），
summary 新增 connectivity_cases 与 judged_pass_rate（无判定型用例时为 null）。
对比报告同步标注且连通用例按引擎口径计通过；总通过率口径不变（ADR-0002）。
This commit is contained in:
sinohqb 2026-07-29 10:32:56 +08:00
parent 5dd1bc8535
commit 8a526599ab
4 changed files with 167 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.scratch/v0.5/issues/02-connectivity-case-annotation.md
View File

@ -4,10 +4,10 @@
**Blocked by:** None — can start immediately.
**Status:** ready-for-agent
**Status:** done
- [ ] 单次报告中连通用例带明确标记，前端可见
- [ ] 对比报告中连通用例同样标注
- [ ] summary 含连通用例数与判定型通过率；全为连通用例时判定型通过率不除零
- [ ] 总通过率数值与升级前一致（口径未变）
- [ ] 报告单元测试覆盖标注与判定型通过率计算（先例：现有报告测试）
- [x] 单次报告中连通用例带明确标记，前端可见
- [x] 对比报告中连通用例同样标注
- [x] summary 含连通用例数与判定型通过率；全为连通用例时判定型通过率不除零
- [x] 总通过率数值与升级前一致（口径未变）
- [x] 报告单元测试覆盖标注与判定型通过率计算（先例：现有报告测试）

47
backend/agenteval/evaluation/report.py
View File

@ -111,9 +111,11 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
# Group by case
case_map: dict[str, dict[str, Any]] = {}
for turn in turns:
case_map.setdefault(turn.case_id, {"turns": [], "results": []})
case_map.setdefault(turn.case_id, {"turns": [], "results": [], "all_replied": True})
sent = turn.get_sent_message()
reply = turn.get_reply()
if reply is None:
case_map[turn.case_id]["all_replied"] = False
case_map[turn.case_id]["turns"].append(
{
"round": turn.round_index,
@ -125,7 +127,7 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
)
for result in results:
case_map.setdefault(result.case_id, {"turns": [], "results": []})
case_map.setdefault(result.case_id, {"turns": [], "results": [], "all_replied": True})
case_map[result.case_id]["results"].append(
{
"rule_type": result.rule_type,
@ -135,18 +137,35 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
}
)
summary = run.summary or {}
errored_case_ids = {e.get("case_id") for e in summary.get("case_errors", [])}
cases = []
for case_id in sorted(case_map.keys()):
item = case_map[case_id]
# 连通用例：无任何判定结果，且每轮都收到回复、无用例级错误（CONTEXT.md）
connectivity = (
not item["results"]
and bool(item["turns"])
and item["all_replied"]
and case_id not in errored_case_ids
)
cases.append(
{
"case_id": case_id,
"connectivity": connectivity,
"turns": sorted(item["turns"], key=lambda x: x["round"]),
"results": item["results"],
}
)
summary = run.summary or {}
total_cases = summary.get("total_cases", 0)
passed_cases = summary.get("passed_cases", 0)
connectivity_count = sum(1 for c in cases if c["connectivity"])
judged_total = total_cases - connectivity_count
# 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数
judged_pass_rate = round((passed_cases - connectivity_count) / judged_total, 4) if judged_total > 0 else None
return {
"run_id": run.id,
"target_id": run.target_id,
@ -157,12 +176,14 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
"started_at": iso_utc(run.started_at),
"completed_at": iso_utc(run.completed_at),
"summary": {
"total_cases": summary.get("total_cases", 0),
"passed_cases": summary.get("passed_cases", 0),
"total_cases": total_cases,
"passed_cases": passed_cases,
"failed_cases": summary.get("failed_cases", 0),
"total_rules": summary.get("total_rules", 0),
"passed_rules": summary.get("passed_rules", 0),
"pass_rate": summary.get("pass_rate", 0.0),
"connectivity_cases": connectivity_count,
"judged_pass_rate": judged_pass_rate,
},
"cases": cases,
}
@ -194,6 +215,9 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[
def _case_passed(c):
if not c:
return None
if c.get("connectivity"):
# 连通用例收到回复即通过（引擎口径）
return True
results = c.get("results", [])
if not results:
# No rule results (e.g. errored case) must not count as passed.
@ -203,6 +227,7 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[
case_diffs.append(
{
"case_id": cid,
"connectivity": bool((ca and ca.get("connectivity")) or (cb and cb.get("connectivity"))),
"run_a_passed": _case_passed(ca),
"run_b_passed": _case_passed(cb),
"changed": _case_passed(ca) != _case_passed(cb),
@ -242,6 +267,8 @@ def render_markdown_report(run_id: str, session=None) -> str:
"""Render a report as Markdown string."""
report = generate_report(run_id, session)
s = report["summary"]
judged_rate = s.get("judged_pass_rate")
judged_rate_text = "—" if judged_rate is None else f"{judged_rate * 100:.1f}%"
lines: list[str] = [
f"# 评测报告 — {report.get('scenario_name', run_id)}",
"",
@ -261,15 +288,23 @@ def render_markdown_report(run_id: str, session=None) -> str:
f"| 总规则数 | {s['total_rules']} |",
f"| 通过规则 | {s['passed_rules']} |",
f"| 通过率 | {s['pass_rate'] * 100:.1f}% |",
f"| 连通用例 | {s.get('connectivity_cases', 0)} |",
f"| 判定型通过率 | {judged_rate_text} |",
"",
"## 用例明细",
"",
]
for case in report.get("cases", []):
if case.get("connectivity"):
badge = "🔗"
else:
case_passed = all(r["passed"] for r in case.get("results", []))
badge = "✅" if case_passed else "❌"
lines.append(f"### {badge} 用例 `{case['case_id']}`")
title = f"### {badge} 用例 `{case['case_id']}`"
if case.get("connectivity"):
title += "（连通用例，未配置判定标准）"
lines.append(title)
lines.append("")
for turn in case.get("turns", []):

19
frontend/web/src/pages/Reports.tsx
View File

@ -1,7 +1,7 @@
import { useEffect, useMemo, useState } from 'react'
import { useSearchParams } from 'react-router-dom'
import {
Button, Card, Col, Collapse, Descriptions, Empty, Row, Segmented,
Alert, Button, Card, Col, Collapse, Descriptions, Empty, Row, Segmented,
Select, Space, Spin, Statistic, Table, Tag, Tooltip, Badge, message,
} from 'antd'
import {
@ -29,6 +29,7 @@ interface RuleResultData {
interface CaseReport {
case_id: string
connectivity: boolean
turns: TurnData[]
results: RuleResultData[]
}
@ -47,6 +48,8 @@ interface Report {
total_rules: number
passed_rules: number
pass_rate: number
connectivity_cases: number
judged_pass_rate: number | null
}
cases: CaseReport[]
}
@ -57,6 +60,7 @@ interface CompareResult {
delta: { pass_rate: number; passed_cases: number; passed_rules: number }
cases: Array<{
case_id: string
connectivity: boolean
run_a_passed: boolean | null
run_b_passed: boolean | null
changed: boolean
@ -364,6 +368,14 @@ function SingleReportView({ report }: { report: Report | null }) {
</Col>
</Row>
{report.summary.connectivity_cases > 0 && (
<Alert type="info" showIcon style={{ marginBottom: 16 }}
message={`本次运行含 ${report.summary.connectivity_cases} 个连通用例（未配置判定标准，收到回复即通过）` +
(report.summary.judged_pass_rate != null
? `，判定型通过率 ${(report.summary.judged_pass_rate * 100).toFixed(1)}%`
: '，无判定型用例')} />
)}
<Card style={{ marginBottom: 16 }}>
<Descriptions size="small" column={2}>
<Descriptions.Item label="评测对象">{report.target_name}</Descriptions.Item>
@ -379,7 +391,9 @@ function SingleReportView({ report }: { report: Report | null }) {
label: (
<Space>
<span style={{ fontWeight: 500 }}>{c.case_id}</span>
{c.results.every((r) => r.passed)
{c.connectivity
? <Tag color="blue">连通用例</Tag>
: c.results.every((r) => r.passed)
? <Tag color="success">全部通过</Tag>
: <Tag color="error">存在失败</Tag>}
<Tag>{c.turns.length} 轮对话</Tag>
@ -501,6 +515,7 @@ function CompareView({ result }: { result: CompareResult | null }) {
<Space>
{r.changed && <Badge dot color="orange" />}
<span style={{ fontWeight: r.changed ? 600 : 400 }}>{id}</span>
{r.connectivity && <Tag color="blue">连通</Tag>}
</Space>
),
},

105
tests/unit/test_report.py
View File

@ -40,8 +40,14 @@ def _seed_run(
pass_rate: float = 1.0,
n_cases: int = 1,
scenario_id: str | None = None,
connectivity_cases: int = 0,
errored_cases: int = 0,
) -> str:
"""Create a minimal completed run with real data in the DB and return run_id."""
"""Create a minimal completed run with real data in the DB and return run_id.
connectivity_cases: extra cases with replied turns but no rule results.
errored_cases: extra cases with a missing-reply turn and no rule results.
"""
target = EvalTarget(
name="测试对象",
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
@ -54,7 +60,11 @@ def _seed_run(
if scenario_id is None:
scenario = Scenario(
name="测试场景",
cases=[Case(id=f"c{i}", type=CaseType.SINGLE, messages=["hi"]) for i in range(n_cases)],
cases=(
[Case(id=f"c{i}", type=CaseType.SINGLE, messages=["hi"]) for i in range(n_cases)]
+ [Case(id=f"conn{i}", type=CaseType.SINGLE, messages=["ping"]) for i in range(connectivity_cases)]
+ [Case(id=f"err{i}", type=CaseType.SINGLE, messages=["ping"]) for i in range(errored_cases)]
),
)
scenario = ScenarioRepository(session).create(scenario)
scenario_id = scenario.id
@ -95,13 +105,35 @@ def _seed_run(
)
result_repo.save_result(eval_result)
for i in range(connectivity_cases):
result_repo.save_turn(Turn(
run_id=run.id,
case_id=f"conn{i}",
round_index=1,
sent_message={"msgBody": {"content": "ping"}},
reply={"msgBody": {"content": "pong"}},
latency_ms=100,
))
for i in range(errored_cases):
result_repo.save_turn(Turn(
run_id=run.id,
case_id=f"err{i}",
round_index=1,
sent_message={"msgBody": {"content": "ping"}},
reply=None,
latency_ms=None,
))
all_total = total + connectivity_cases + errored_cases
all_passed = passed + connectivity_cases # 连通用例收到回复即通过（引擎口径）
run.summary = {
"total_cases": total,
"passed_cases": passed,
"failed_cases": total - passed,
"total_cases": all_total,
"passed_cases": all_passed,
"failed_cases": all_total - all_passed,
"total_rules": total,
"passed_rules": passed,
"pass_rate": round(pass_rate, 4),
"pass_rate": round(all_passed / all_total, 4) if all_total else 0.0,
}
RunRepository(session).update(run)
return run.id
@ -210,6 +242,67 @@ def test_compare_report_different_scenarios_rejected(report_session):
generate_compare_report(run_id_a, run_id_b, report_session)
# ── connectivity case annotation (ticket 02) ─────────────────────────────
def test_report_marks_connectivity_case(report_session):
run_id = _seed_run(report_session, pass_rate=0.5, n_cases=2, connectivity_cases=1)
report = generate_report(run_id, report_session)
by_id = {c["case_id"]: c for c in report["cases"]}
assert by_id["conn0"]["connectivity"] is True
assert by_id["c0"]["connectivity"] is False
assert by_id["c1"]["connectivity"] is False
s = report["summary"]
assert s["connectivity_cases"] == 1
# 判定型通过率 = 判定型通过 1 ÷ 判定型总数 2
assert s["judged_pass_rate"] == 0.5
# 总通过率口径不变（含连通用例，ADR-0002）
assert s["pass_rate"] == round(2 / 3, 4)
def test_errored_case_not_marked_connectivity(report_session):
"""无判定结果但缺回复的用例是执行失败，不是连通用例。"""
run_id = _seed_run(report_session, n_cases=1, errored_cases=1)
report = generate_report(run_id, report_session)
by_id = {c["case_id"]: c for c in report["cases"]}
assert by_id["err0"]["connectivity"] is False
assert report["summary"]["connectivity_cases"] == 0
def test_all_connectivity_judged_pass_rate_none(report_session):
"""全为连通用例时判定型通过率为 None，不除零。"""
run_id = _seed_run(report_session, n_cases=0, connectivity_cases=2)
report = generate_report(run_id, report_session)
s = report["summary"]
assert s["connectivity_cases"] == 2
assert s["judged_pass_rate"] is None
def test_compare_report_marks_connectivity(report_session):
run_id_a = _seed_run(report_session, n_cases=1, connectivity_cases=1)
sid = _scenario_of(report_session, run_id_a)
run_id_b = _seed_run(report_session, n_cases=1, connectivity_cases=1, scenario_id=sid)
result = generate_compare_report(run_id_a, run_id_b, report_session)
by_id = {c["case_id"]: c for c in result["cases"]}
assert by_id["conn0"]["connectivity"] is True
assert by_id["c0"]["connectivity"] is False
# 连通用例双方均按引擎口径视为通过，不应标记 changed
assert by_id["conn0"]["run_a_passed"] is True
assert by_id["conn0"]["run_b_passed"] is True
assert by_id["conn0"]["changed"] is False
def test_markdown_report_shows_connectivity(report_session):
run_id = _seed_run(report_session, n_cases=1, connectivity_cases=1)
md = render_markdown_report(run_id, report_session)
assert "连通用例" in md
assert "判定型通过率" in md
# ── render_markdown_report ────────────────────────────────────────────────
def test_render_markdown_contains_header(report_session):