5db0ede4f4
Some checks failed
CI / test (push) Failing after 50s
「用例是否通过」此前散落 8 处且互相矛盾：engine 权威判定焊死在持久化里 不可单测；report 聚合/compare/markdown 各自从规则结果反推，规则还不一致 （markdown 用 all([]) 把故障用例误渲染成 ✅）。 - 新增纯函数 evaluation/judgement.combine_case_outcome（RuleOutcome/ CaseOutcome），判定组合脱离通道与 DB 可单测（判定矩阵 14 例） - engine 调用它一次，逐用例权威结果写入 summary.case_outcomes（JSON， 零迁移）；report/compare/markdown 只读权威值，老 run fallback 反推 - 故障用例判 False（ADR-0002）：修正 markdown 的 ✅ bug 与 compare 的 None；顺带修 engine 连通用例无回复也算通过的 bug - pass_rate 口径改为用例级（CONTEXT.md 词条），规则级保留在 passed_rules/total_rules；CLI 对比标签同步更正 - 修 RunRepository.update 漏拷 scenario_version/triggered_by 的字段漂移
387 lines
16 KiB
Python
387 lines
16 KiB
Python
"""Report generation for evaluation runs."""
|
||
|
||
import json
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any, Optional
|
||
|
||
from jinja2 import Template
|
||
|
||
from agenteval.storage.db import DATA_DIR, iso_utc
|
||
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
||
from agenteval.utils.llm import extract_reply_text
|
||
|
||
HTML_TEMPLATE = """<!DOCTYPE html>
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<title>评测报告 - {{ report.run_id }}</title>
|
||
<style>
|
||
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; margin: 40px; background: #f5f5f5; }
|
||
.container { max-width: 960px; margin: 0 auto; background: #fff; padding: 32px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.05); }
|
||
h1 { margin-top: 0; }
|
||
.summary { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 16px; margin: 24px 0; }
|
||
.card { background: #fafafa; border-radius: 6px; padding: 16px; text-align: center; }
|
||
.card .value { font-size: 24px; font-weight: 700; }
|
||
.card .label { color: #666; font-size: 14px; margin-top: 4px; }
|
||
.case { border: 1px solid #e0e0e0; border-radius: 6px; margin: 16px 0; padding: 16px; }
|
||
.case-title { font-weight: 600; margin-bottom: 8px; }
|
||
.turn { background: #f9f9f9; border-radius: 4px; padding: 12px; margin: 8px 0; }
|
||
.message-label { color: #666; font-size: 12px; }
|
||
.rule { display: flex; align-items: center; gap: 8px; margin: 6px 0; }
|
||
.badge { padding: 2px 8px; border-radius: 4px; font-size: 12px; }
|
||
.pass { background: #e6f7e6; color: #2e7d32; }
|
||
.fail { background: #ffebee; color: #c62828; }
|
||
pre { white-space: pre-wrap; word-break: break-word; background: #f5f5f5; padding: 8px; border-radius: 4px; }
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<div class="container">
|
||
<h1>评测报告</h1>
|
||
<p>评测对象：{{ report.target_name }}（{{ report.target_id }}）</p>
|
||
<p>评测场景：{{ report.scenario_name }}（{{ report.scenario_id }}）</p>
|
||
<p>执行时间：{{ report.started_at }} 至 {{ report.completed_at or '进行中' }}</p>
|
||
|
||
<div class="summary">
|
||
<div class="card">
|
||
<div class="value">{{ report.summary.total_cases }}</div>
|
||
<div class="label">用例总数</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="value">{{ report.summary.passed_cases }}</div>
|
||
<div class="label">通过用例</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="value">{{ report.summary.total_rules }}</div>
|
||
<div class="label">规则总数</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="value">{{ "%.2f"|format(report.summary.pass_rate * 100) }}%</div>
|
||
<div class="label">规则通过率</div>
|
||
</div>
|
||
</div>
|
||
|
||
{% for case in report.cases %}
|
||
<div class="case">
|
||
<div class="case-title">用例 {{ case.case_id }}</div>
|
||
{% for turn in case.turns %}
|
||
<div class="turn">
|
||
<div class="message-label">用户消息</div>
|
||
<pre>{{ turn.sent_text }}</pre>
|
||
<div class="message-label">智能体回复（{{ turn.latency_ms }}ms）</div>
|
||
<pre>{{ turn.reply_text or '（无回复）' }}</pre>
|
||
</div>
|
||
{% endfor %}
|
||
<div>
|
||
{% for result in case.results %}
|
||
<div class="rule">
|
||
<span class="badge {{ 'pass' if result.passed else 'fail' }}">{{ '通过' if result.passed else '失败' }}</span>
|
||
<span>{{ result.rule_type }}: {{ result.reason }}</span>
|
||
</div>
|
||
{% endfor %}
|
||
</div>
|
||
</div>
|
||
{% endfor %}
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
|
||
def _extract_text(data: Any) -> str:
|
||
return extract_reply_text(data)
|
||
|
||
|
||
def generate_report(run_id: str, session=None) -> dict[str, Any]:
|
||
"""Build a structured report dict for a run."""
|
||
run_repo = RunRepository(session)
|
||
target_repo = TargetRepository(session)
|
||
scenario_repo = ScenarioRepository(session)
|
||
|
||
run = run_repo.get(run_id)
|
||
if not run:
|
||
raise ValueError(f"run not found: {run_id}")
|
||
|
||
target = target_repo.get(run.target_id)
|
||
scenario = scenario_repo.get(run.scenario_id)
|
||
|
||
turns = run_repo.get_turns(run_id)
|
||
results = run_repo.get_results(run_id)
|
||
|
||
# Group by case
|
||
case_map: dict[str, dict[str, Any]] = {}
|
||
for turn in turns:
|
||
case_map.setdefault(turn.case_id, {"turns": [], "results": [], "all_replied": True})
|
||
sent = turn.get_sent_message()
|
||
reply = turn.get_reply()
|
||
if reply is None:
|
||
case_map[turn.case_id]["all_replied"] = False
|
||
case_map[turn.case_id]["turns"].append(
|
||
{
|
||
"round": turn.round_index,
|
||
"sent_text": _extract_text(sent.get("msgBody")),
|
||
"reply_text": _extract_text(reply.get("msgBody") if reply else None),
|
||
"latency_ms": turn.latency_ms,
|
||
"question_msg_id": turn.question_msg_id,
|
||
}
|
||
)
|
||
|
||
for result in results:
|
||
case_map.setdefault(result.case_id, {"turns": [], "results": [], "all_replied": True})
|
||
case_map[result.case_id]["results"].append(
|
||
{
|
||
"rule_type": result.rule_type,
|
||
"passed": result.passed,
|
||
"score": result.score,
|
||
"reason": result.reason,
|
||
}
|
||
)
|
||
|
||
summary = run.summary or {}
|
||
errored_case_ids = {e.get("case_id") for e in summary.get("case_errors", [])}
|
||
# 权威判定：引擎经 judgement.combine_case_outcome 算一次写入 summary；
|
||
# 老 run 没有该字段时退回从持久化结果反推（WEIGHTED/ANY 只能近似）。
|
||
authoritative: dict[str, Any] = summary.get("case_outcomes", {})
|
||
|
||
cases = []
|
||
for case_id in sorted(case_map.keys()):
|
||
item = case_map[case_id]
|
||
if case_id in authoritative:
|
||
outcome = authoritative[case_id]
|
||
connectivity = bool(outcome.get("connectivity"))
|
||
passed = bool(outcome.get("passed"))
|
||
else:
|
||
# 连通用例：无任何判定结果，且每轮都收到回复、无用例级错误（CONTEXT.md）
|
||
connectivity = (
|
||
not item["results"]
|
||
and bool(item["turns"])
|
||
and item["all_replied"]
|
||
and case_id not in errored_case_ids
|
||
)
|
||
if connectivity:
|
||
passed = True
|
||
elif not item["results"]:
|
||
# 故障用例（无结果且非连通）＝不通过（ADR-0002）
|
||
passed = False
|
||
else:
|
||
passed = all(r["passed"] for r in item["results"])
|
||
cases.append(
|
||
{
|
||
"case_id": case_id,
|
||
"passed": passed,
|
||
"connectivity": connectivity,
|
||
"turns": sorted(item["turns"], key=lambda x: x["round"]),
|
||
"results": item["results"],
|
||
}
|
||
)
|
||
|
||
total_cases = summary.get("total_cases", 0)
|
||
passed_cases = summary.get("passed_cases", 0)
|
||
connectivity_count = sum(1 for c in cases if c["connectivity"])
|
||
judged_total = total_cases - connectivity_count
|
||
# 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数
|
||
judged_pass_rate = round((passed_cases - connectivity_count) / judged_total, 4) if judged_total > 0 else None
|
||
|
||
return {
|
||
"run_id": run.id,
|
||
"target_id": run.target_id,
|
||
"target_name": target.name if target else "未知",
|
||
"scenario_id": run.scenario_id,
|
||
"scenario_name": scenario.name if scenario else "未知",
|
||
"scenario_version": run.scenario_version,
|
||
"triggered_by": run.triggered_by.value,
|
||
"status": run.status.value,
|
||
"started_at": iso_utc(run.started_at),
|
||
"completed_at": iso_utc(run.completed_at),
|
||
"summary": {
|
||
"total_cases": total_cases,
|
||
"passed_cases": passed_cases,
|
||
"failed_cases": summary.get("failed_cases", 0),
|
||
"total_rules": summary.get("total_rules", 0),
|
||
"passed_rules": summary.get("passed_rules", 0),
|
||
"pass_rate": summary.get("pass_rate", 0.0),
|
||
"connectivity_cases": connectivity_count,
|
||
"judged_pass_rate": judged_pass_rate,
|
||
},
|
||
"cases": cases,
|
||
}
|
||
|
||
|
||
def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[str, Any]:
|
||
"""Build a side-by-side comparison dict for two runs of the same scenario."""
|
||
report_a = generate_report(run_id_1, session)
|
||
report_b = generate_report(run_id_2, session)
|
||
|
||
# Cross-scenario case_ids never overlap, so every case would be flagged
|
||
# "changed" and the diff would be meaningless — reject early.
|
||
if report_a.get("scenario_id") != report_b.get("scenario_id"):
|
||
raise ValueError("compare report requires both runs to use the same scenario")
|
||
# 同场景还须同考纲版本才可比（ADR-0001）
|
||
if report_a.get("scenario_version") != report_b.get("scenario_version"):
|
||
raise ValueError(
|
||
"compare report requires the same scenario version "
|
||
f"(A: v{report_a.get('scenario_version')}, B: v{report_b.get('scenario_version')})"
|
||
)
|
||
|
||
def _summary_delta(key: str) -> float:
|
||
return report_b["summary"][key] - report_a["summary"][key]
|
||
|
||
# Case-level diff: match by case_id
|
||
cases_a = {c["case_id"]: c for c in report_a.get("cases", [])}
|
||
cases_b = {c["case_id"]: c for c in report_b.get("cases", [])}
|
||
all_case_ids = sorted(set(cases_a) | set(cases_b))
|
||
|
||
case_diffs = []
|
||
for cid in all_case_ids:
|
||
ca = cases_a.get(cid)
|
||
cb = cases_b.get(cid)
|
||
|
||
def _case_passed(c):
|
||
# None 仅表示该 run 没有这个用例；判定本身读权威 passed 字段
|
||
return c["passed"] if c else None
|
||
|
||
case_diffs.append(
|
||
{
|
||
"case_id": cid,
|
||
"connectivity": bool((ca and ca.get("connectivity")) or (cb and cb.get("connectivity"))),
|
||
"run_a_passed": _case_passed(ca),
|
||
"run_b_passed": _case_passed(cb),
|
||
"changed": _case_passed(ca) != _case_passed(cb),
|
||
"run_a_results": ca["results"] if ca else [],
|
||
"run_b_results": cb["results"] if cb else [],
|
||
}
|
||
)
|
||
|
||
return {
|
||
"run_a": {
|
||
"run_id": run_id_1,
|
||
"target_name": report_a.get("target_name"),
|
||
"scenario_name": report_a.get("scenario_name"),
|
||
"scenario_version": report_a.get("scenario_version"),
|
||
"triggered_by": report_a.get("triggered_by"),
|
||
"status": report_a.get("status"),
|
||
"started_at": report_a.get("started_at"),
|
||
"summary": report_a["summary"],
|
||
},
|
||
"run_b": {
|
||
"run_id": run_id_2,
|
||
"target_name": report_b.get("target_name"),
|
||
"scenario_name": report_b.get("scenario_name"),
|
||
"scenario_version": report_b.get("scenario_version"),
|
||
"triggered_by": report_b.get("triggered_by"),
|
||
"status": report_b.get("status"),
|
||
"started_at": report_b.get("started_at"),
|
||
"summary": report_b["summary"],
|
||
},
|
||
"delta": {
|
||
"pass_rate": round(_summary_delta("pass_rate"), 4),
|
||
"passed_cases": int(_summary_delta("passed_cases")),
|
||
"passed_rules": int(_summary_delta("passed_rules")),
|
||
},
|
||
"cases": case_diffs,
|
||
"changed_cases": sum(1 for c in case_diffs if c["changed"]),
|
||
}
|
||
|
||
|
||
def render_markdown_report(run_id: str, session=None) -> str:
|
||
"""Render a report as Markdown string."""
|
||
report = generate_report(run_id, session)
|
||
s = report["summary"]
|
||
judged_rate = s.get("judged_pass_rate")
|
||
judged_rate_text = "—" if judged_rate is None else f"{judged_rate * 100:.1f}%"
|
||
lines: list[str] = [
|
||
f"# 评测报告 — {report.get('scenario_name', run_id)}",
|
||
"",
|
||
f"**评测对象**: {report.get('target_name', '-')} ",
|
||
f"**评测场景**: {report.get('scenario_name', '-')} ",
|
||
f"**状态**: {report.get('status', '-')} ",
|
||
f"**开始时间**: {report.get('started_at', '-')} ",
|
||
f"**完成时间**: {report.get('completed_at', '-')} ",
|
||
"",
|
||
"## 汇总",
|
||
"",
|
||
"| 指标 | 数值 |",
|
||
"|------|------|",
|
||
f"| 总用例数 | {s['total_cases']} |",
|
||
f"| 通过用例 | {s['passed_cases']} |",
|
||
f"| 失败用例 | {s['failed_cases']} |",
|
||
f"| 总规则数 | {s['total_rules']} |",
|
||
f"| 通过规则 | {s['passed_rules']} |",
|
||
f"| 通过率 | {s['pass_rate'] * 100:.1f}% |",
|
||
f"| 连通用例 | {s.get('connectivity_cases', 0)} |",
|
||
f"| 判定型通过率 | {judged_rate_text} |",
|
||
"",
|
||
"## 用例明细",
|
||
"",
|
||
]
|
||
|
||
for case in report.get("cases", []):
|
||
if case.get("connectivity"):
|
||
badge = "🔗"
|
||
else:
|
||
badge = "✅" if case.get("passed") else "❌"
|
||
title = f"### {badge} 用例 `{case['case_id']}`"
|
||
if case.get("connectivity"):
|
||
title += "（连通用例，未配置判定标准）"
|
||
lines.append(title)
|
||
lines.append("")
|
||
|
||
for turn in case.get("turns", []):
|
||
lines.append(f"**第 {turn['round']} 轮**")
|
||
lines.append("")
|
||
lines.append(f"> **用户**: {turn.get('sent_text', '—')}")
|
||
lines.append("")
|
||
reply = turn.get("reply_text") or "（无回复）"
|
||
lines.append(f"> **智能体**: {reply}")
|
||
if turn.get("latency_ms") is not None:
|
||
lines.append(f"> *延迟: {turn['latency_ms']}ms*")
|
||
lines.append("")
|
||
|
||
if case.get("results"):
|
||
lines.append("**规则评估结果**")
|
||
lines.append("")
|
||
lines.append("| 规则 | 结果 | 评分 | 说明 |")
|
||
lines.append("|------|------|------|------|")
|
||
for r in case["results"]:
|
||
badge = "✅" if r["passed"] else "❌"
|
||
score = f"{r['score']:.2f}" if r.get("score") is not None else "-"
|
||
lines.append(f"| {r['rule_type']} | {badge} | {score} | {r.get('reason', '')} |")
|
||
lines.append("")
|
||
|
||
return "\n".join(lines)
|
||
|
||
|
||
def render_json_report(run_id: str, session=None) -> str:
|
||
"""Render a report as JSON string."""
|
||
report = generate_report(run_id, session)
|
||
return json.dumps(report, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def render_html_report(run_id: str, session=None) -> str:
|
||
"""Render a report as HTML string."""
|
||
report = generate_report(run_id, session)
|
||
template = Template(HTML_TEMPLATE)
|
||
return template.render(report=report)
|
||
|
||
|
||
def save_report(run_id: str, fmt: str = "html", output_dir: Optional[Path] = None) -> Path:
|
||
"""Generate and save a report to disk."""
|
||
output_dir = output_dir or DATA_DIR / "reports"
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
if fmt == "html":
|
||
content = render_html_report(run_id)
|
||
path = output_dir / f"report_{run_id}_{timestamp}.html"
|
||
elif fmt == "json":
|
||
content = render_json_report(run_id)
|
||
path = output_dir / f"report_{run_id}_{timestamp}.json"
|
||
elif fmt == "markdown":
|
||
content = render_markdown_report(run_id)
|
||
path = output_dir / f"report_{run_id}_{timestamp}.md"
|
||
else:
|
||
raise ValueError(f"unsupported report format: {fmt}")
|
||
|
||
path.write_text(content, encoding="utf-8")
|
||
return path
|