solahqb/AgentEvalTool
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor(report): split report generation from pure rendering

Browse Source 
report.py mixed DB-reading generation with string formatting: the four
render_*_report(run_id, session) functions each re-fetched via
generate_report, so the HTML/Markdown/JSON formatting was welded to storage
and could not be unit-tested from a plain dict. Extract the formatting into a
new pure report_render module whose renderers take the already-built report
dict (no session, no storage import). Migrate every caller to generate-then-
render, delete the old coupled renderers with no back-compat shim, and drop
the _aggregate_runs middle-man alias in favour of metrics.aggregate_runs.
This commit is contained in:
sinohqb 2026-07-31 10:19:04 +08:00
parent 0cca4963d1
commit f285738f6d
8 changed files with 456 additions and 278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

266
backend/agenteval/evaluation/report.py
View File

@ -1,95 +1,21 @@
"""Report generation for evaluation runs."""
"""Report generation for evaluation runs.
Generation only: read the DB / model objects and build the report dict.
Formatting lives in ``report_render`` (pure dict → HTML/Markdown/JSON).
"""
import json
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional
from jinja2 import Template
from agenteval.evaluation.metrics import aggregate_runs
from agenteval.evaluation.report_render import render_html, render_json, render_markdown
from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary
from agenteval.storage.db import DATA_DIR, iso_utc
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
from agenteval.utils.llm import extract_reply_text
HTML_TEMPLATE = """<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>评测报告 - {{ report.run_id }}</title>
<style>
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; margin: 40px; background: #f5f5f5; }
.container { max-width: 960px; margin: 0 auto; background: #fff; padding: 32px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.05); }
h1 { margin-top: 0; }
.summary { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 16px; margin: 24px 0; }
.card { background: #fafafa; border-radius: 6px; padding: 16px; text-align: center; }
.card .value { font-size: 24px; font-weight: 700; }
.card .label { color: #666; font-size: 14px; margin-top: 4px; }
.case { border: 1px solid #e0e0e0; border-radius: 6px; margin: 16px 0; padding: 16px; }
.case-title { font-weight: 600; margin-bottom: 8px; }
.turn { background: #f9f9f9; border-radius: 4px; padding: 12px; margin: 8px 0; }
.message-label { color: #666; font-size: 12px; }
.rule { display: flex; align-items: center; gap: 8px; margin: 6px 0; }
.badge { padding: 2px 8px; border-radius: 4px; font-size: 12px; }
.pass { background: #e6f7e6; color: #2e7d32; }
.fail { background: #ffebee; color: #c62828; }
pre { white-space: pre-wrap; word-break: break-word; background: #f5f5f5; padding: 8px; border-radius: 4px; }
</style>
</head>
<body>
<div class="container">
<h1>评测报告</h1>
<p>评测对象：{{ report.target_name }}（{{ report.target_id }}）</p>
<p>评测场景：{{ report.scenario_name }}（{{ report.scenario_id }}）</p>
<p>执行时间：{{ report.started_at }} 至 {{ report.completed_at or '进行中' }}</p>
<div class="summary">
<div class="card">
<div class="value">{{ report.summary.total_cases }}</div>
<div class="label">用例总数</div>
</div>
<div class="card">
<div class="value">{{ report.summary.passed_cases }}</div>
<div class="label">通过用例</div>
</div>
<div class="card">
<div class="value">{{ report.summary.total_rules }}</div>
<div class="label">规则总数</div>
</div>
<div class="card">
<div class="value">{{ "%.2f"|format(report.summary.pass_rate * 100) }}%</div>
<div class="label">规则通过率</div>
</div>
</div>
{% for case in report.cases %}
<div class="case">
<div class="case-title">用例 {{ case.case_id }}</div>
{% for turn in case.turns %}
<div class="turn">
<div class="message-label">用户消息</div>
<pre>{{ turn.sent_text }}</pre>
<div class="message-label">智能体回复（{{ turn.latency_ms }}ms）</div>
<pre>{{ turn.reply_text or '（无回复）' }}</pre>
</div>
{% endfor %}
<div>
{% for result in case.results %}
<div class="rule">
<span class="badge {{ 'pass' if result.passed else 'fail' }}">{{ '通过' if result.passed else '失败' }}</span>
<span>{{ result.rule_type }}: {{ result.reason }}</span>
</div>
{% endfor %}
</div>
</div>
{% endfor %}
</div>
</body>
</html>
"""
def _extract_text(data: Any) -> str:
return extract_reply_text(data)
@ -292,24 +218,19 @@ def _to_utc(dt: Optional[datetime]) -> Optional[datetime]:
return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt
def _aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]:
"""Delegates to the single cross-run aggregation seam (ADR-0004)."""
return aggregate_runs(runs)
def summarize_campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict[str, Any]:
"""Compact list-row progress: completed vs *planned* total, plus pass_rate.
Unlike ``campaign_progress`` (live window position), this powers the list
view. ``planned_total`` is the sum of plan-entry counts — a fixed target the
campaign works toward, so the progress bar fills from 0 rather than tracking
a growing spawned count. ``overall_pass_rate`` reuses ``_aggregate_runs`` so
a growing spawned count. ``overall_pass_rate`` reuses ``aggregate_runs`` so
the list figure matches the report exactly (ADR-0002: failures count as 0.0).
"""
return {
"completed_runs": sum(1 for r in runs if r.status == RunStatus.COMPLETED),
"planned_total": sum(entry.count for entry in campaign.plan),
"overall_pass_rate": _aggregate_runs(runs)["pass_rate"],
"overall_pass_rate": aggregate_runs(runs)["pass_rate"],
}
@ -350,7 +271,7 @@ def generate_campaign_report(
time_trend = []
for idx in range(bucket_count):
agg = _aggregate_runs(buckets.get(idx, []))
agg = aggregate_runs(buckets.get(idx, []))
time_trend.append({
"bucket_index": idx,
"start_seconds": round(idx * bucket_seconds, 3),
@ -364,7 +285,7 @@ def generate_campaign_report(
by_scenario[run.scenario_id].append(run)
capability_summary = []
for sid, sruns in by_scenario.items():
agg = _aggregate_runs(sruns)
agg = aggregate_runs(sruns)
capability_summary.append({
"scenario_id": sid,
"scenario_name": scenario_names.get(sid, (sid or "")[:8]),
@ -372,7 +293,7 @@ def generate_campaign_report(
})
capability_summary.sort(key=lambda s: s["run_count"], reverse=True)
overall = _aggregate_runs(runs)
overall = aggregate_runs(runs)
return {
"campaign_id": campaign.id,
"name": campaign.name,
@ -394,163 +315,22 @@ def generate_campaign_report(
}
def render_campaign_markdown_report(
campaign: Campaign,
runs: list[EvalRun],
*,
scenario_names: Optional[dict[str, str]] = None,
) -> str:
"""Render the dual-axis campaign report as Markdown (reuses the export path)."""
report = generate_campaign_report(campaign, runs, scenario_names=scenario_names)
s = report["summary"]
def _pct(v: Optional[float]) -> str:
return "—" if v is None else f"{v * 100:.1f}%"
def _ms(v: Optional[float]) -> str:
return "—" if v is None else f"{v:.0f}ms"
lines: list[str] = [
f"# 活动周期报告 — {report['name']}",
"",
f"**状态**: {report['status']} ",
f"**窗口**: {report['window_seconds']}s（倍速 {report['time_scale']}） ",
f"**开始时间**: {report['started_at'] or '-'} ",
f"**完成时间**: {report['completed_at'] or '-'} ",
"",
"## 汇总",
"",
"| 指标 | 数值 |",
"|------|------|",
f"| 子运行总数 | {s['total_runs']} |",
f"| 已完成 | {s['completed_runs']} |",
f"| 整窗通过率 | {_pct(s['overall_pass_rate'])} |",
f"| 整窗可用性 | {_pct(s['overall_availability'])} |",
f"| 平均时延 | {_ms(s['avg_latency_ms'])} |",
"",
"## 时间趋势",
"",
"| 时段(秒) | 运行数 | 通过率 | 可用性 | 时延 |",
"|------|------|------|------|------|",
]
for b in report["time_trend"]:
lines.append(
f"| {b['start_seconds']:.0f}–{b['end_seconds']:.0f} | {b['run_count']} | "
f"{_pct(b['pass_rate'])} | {_pct(b['availability'])} | {_ms(b['avg_latency_ms'])} |"
)
lines += [
"",
"## 能力汇总",
"",
"| 场景 | 运行数 | 通过率 | 可用性 | 时延 |",
"|------|------|------|------|------|",
]
for c in report["capability_summary"]:
lines.append(
f"| {c['scenario_name']} | {c['run_count']} | {_pct(c['pass_rate'])} | "
f"{_pct(c['availability'])} | {_ms(c['avg_latency_ms'])} |"
)
return "\n".join(lines)
def render_markdown_report(run_id: str, session=None) -> str:
"""Render a report as Markdown string."""
report = generate_report(run_id, session)
s = report["summary"]
judged_rate = s.get("judged_pass_rate")
judged_rate_text = "—" if judged_rate is None else f"{judged_rate * 100:.1f}%"
lines: list[str] = [
f"# 评测报告 — {report.get('scenario_name', run_id)}",
"",
f"**评测对象**: {report.get('target_name', '-')} ",
f"**评测场景**: {report.get('scenario_name', '-')} ",
f"**状态**: {report.get('status', '-')} ",
f"**开始时间**: {report.get('started_at', '-')} ",
f"**完成时间**: {report.get('completed_at', '-')} ",
"",
"## 汇总",
"",
"| 指标 | 数值 |",
"|------|------|",
f"| 总用例数 | {s['total_cases']} |",
f"| 通过用例 | {s['passed_cases']} |",
f"| 失败用例 | {s['failed_cases']} |",
f"| 总规则数 | {s['total_rules']} |",
f"| 通过规则 | {s['passed_rules']} |",
f"| 通过率 | {s['pass_rate'] * 100:.1f}% |",
f"| 连通用例 | {s.get('connectivity_cases', 0)} |",
f"| 判定型通过率 | {judged_rate_text} |",
"",
"## 用例明细",
"",
]
for case in report.get("cases", []):
if case.get("connectivity"):
badge = "🔗"
else:
badge = "✅" if case.get("passed") else "❌"
title = f"### {badge} 用例 `{case['case_id']}`"
if case.get("connectivity"):
title += "（连通用例，未配置判定标准）"
lines.append(title)
lines.append("")
for turn in case.get("turns", []):
lines.append(f"**第 {turn['round']} 轮**")
lines.append("")
lines.append(f"> **用户**: {turn.get('sent_text', '—')}")
lines.append("")
reply = turn.get("reply_text") or "（无回复）"
lines.append(f"> **智能体**: {reply}")
if turn.get("latency_ms") is not None:
lines.append(f"> *延迟: {turn['latency_ms']}ms*")
lines.append("")
if case.get("results"):
lines.append("**规则评估结果**")
lines.append("")
lines.append("| 规则 | 结果 | 评分 | 说明 |")
lines.append("|------|------|------|------|")
for r in case["results"]:
badge = "✅" if r["passed"] else "❌"
score = f"{r['score']:.2f}" if r.get("score") is not None else "-"
lines.append(f"| {r['rule_type']} | {badge} | {score} | {r.get('reason', '')} |")
lines.append("")
return "\n".join(lines)
def render_json_report(run_id: str, session=None) -> str:
"""Render a report as JSON string."""
report = generate_report(run_id, session)
return json.dumps(report, ensure_ascii=False, indent=2)
def render_html_report(run_id: str, session=None) -> str:
"""Render a report as HTML string."""
report = generate_report(run_id, session)
template = Template(HTML_TEMPLATE)
return template.render(report=report)
def save_report(run_id: str, fmt: str = "html", output_dir: Optional[Path] = None) -> Path:
"""Generate and save a report to disk."""
"""Generate a run report and save it to disk in the requested format."""
output_dir = output_dir or DATA_DIR / "reports"
output_dir.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
if fmt == "html":
content = render_html_report(run_id)
path = output_dir / f"report_{run_id}_{timestamp}.html"
elif fmt == "json":
content = render_json_report(run_id)
path = output_dir / f"report_{run_id}_{timestamp}.json"
elif fmt == "markdown":
content = render_markdown_report(run_id)
path = output_dir / f"report_{run_id}_{timestamp}.md"
else:
report = generate_report(run_id)
renderers = {
"html": (render_html, "html"),
"json": (render_json, "json"),
"markdown": (render_markdown, "md"),
}
if fmt not in renderers:
raise ValueError(f"unsupported report format: {fmt}")
render, ext = renderers[fmt]
path.write_text(content, encoding="utf-8")
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
path = output_dir / f"report_{run_id}_{timestamp}.{ext}"
path.write_text(render(report), encoding="utf-8")
return path

219
backend/agenteval/evaluation/report_render.py Normal file
View File

@ -0,0 +1,219 @@
"""Pure renderers: report dict in, HTML/Markdown/JSON string out.
No I/O and no storage access — every function takes the dict produced by the
generation side (``evaluation.report``) so rendering can be unit-tested from a
hand-built dict. Report *content* decisions (which cases pass, which rates to
show) belong to generation; this module only formats.
"""
import json
from typing import Any, Optional
from jinja2 import Template
HTML_TEMPLATE = """<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>评测报告 - {{ report.run_id }}</title>
<style>
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; margin: 40px; background: #f5f5f5; }
.container { max-width: 960px; margin: 0 auto; background: #fff; padding: 32px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.05); }
h1 { margin-top: 0; }
.summary { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 16px; margin: 24px 0; }
.card { background: #fafafa; border-radius: 6px; padding: 16px; text-align: center; }
.card .value { font-size: 24px; font-weight: 700; }
.card .label { color: #666; font-size: 14px; margin-top: 4px; }
.case { border: 1px solid #e0e0e0; border-radius: 6px; margin: 16px 0; padding: 16px; }
.case-title { font-weight: 600; margin-bottom: 8px; }
.turn { background: #f9f9f9; border-radius: 4px; padding: 12px; margin: 8px 0; }
.message-label { color: #666; font-size: 12px; }
.rule { display: flex; align-items: center; gap: 8px; margin: 6px 0; }
.badge { padding: 2px 8px; border-radius: 4px; font-size: 12px; }
.pass { background: #e6f7e6; color: #2e7d32; }
.fail { background: #ffebee; color: #c62828; }
pre { white-space: pre-wrap; word-break: break-word; background: #f5f5f5; padding: 8px; border-radius: 4px; }
</style>
</head>
<body>
<div class="container">
<h1>评测报告</h1>
<p>评测对象：{{ report.target_name }}（{{ report.target_id }}）</p>
<p>评测场景：{{ report.scenario_name }}（{{ report.scenario_id }}）</p>
<p>执行时间：{{ report.started_at }} 至 {{ report.completed_at or '进行中' }}</p>
<div class="summary">
<div class="card">
<div class="value">{{ report.summary.total_cases }}</div>
<div class="label">用例总数</div>
</div>
<div class="card">
<div class="value">{{ report.summary.passed_cases }}</div>
<div class="label">通过用例</div>
</div>
<div class="card">
<div class="value">{{ report.summary.total_rules }}</div>
<div class="label">规则总数</div>
</div>
<div class="card">
<div class="value">{{ "%.2f"|format(report.summary.pass_rate * 100) }}%</div>
<div class="label">规则通过率</div>
</div>
</div>
{% for case in report.cases %}
<div class="case">
<div class="case-title">用例 {{ case.case_id }}</div>
{% for turn in case.turns %}
<div class="turn">
<div class="message-label">用户消息</div>
<pre>{{ turn.sent_text }}</pre>
<div class="message-label">智能体回复（{{ turn.latency_ms }}ms）</div>
<pre>{{ turn.reply_text or '（无回复）' }}</pre>
</div>
{% endfor %}
<div>
{% for result in case.results %}
<div class="rule">
<span class="badge {{ 'pass' if result.passed else 'fail' }}">{{ '通过' if result.passed else '失败' }}</span>
<span>{{ result.rule_type }}: {{ result.reason }}</span>
</div>
{% endfor %}
</div>
</div>
{% endfor %}
</div>
</body>
</html>
"""
def _pct(v: Optional[float]) -> str:
return "—" if v is None else f"{v * 100:.1f}%"
def _ms(v: Optional[float]) -> str:
return "—" if v is None else f"{v:.0f}ms"
def render_html(report: dict[str, Any]) -> str:
"""Render a run report dict as an HTML string."""
return Template(HTML_TEMPLATE).render(report=report)
def render_json(report: dict[str, Any]) -> str:
"""Render a report dict as a JSON string."""
return json.dumps(report, ensure_ascii=False, indent=2)
def render_markdown(report: dict[str, Any]) -> str:
"""Render a run report dict as Markdown."""
s = report["summary"]
judged_rate = s.get("judged_pass_rate")
judged_rate_text = "—" if judged_rate is None else f"{judged_rate * 100:.1f}%"
lines: list[str] = [
f"# 评测报告 — {report.get('scenario_name', report.get('run_id', ''))}",
"",
f"**评测对象**: {report.get('target_name', '-')} ",
f"**评测场景**: {report.get('scenario_name', '-')} ",
f"**状态**: {report.get('status', '-')} ",
f"**开始时间**: {report.get('started_at', '-')} ",
f"**完成时间**: {report.get('completed_at', '-')} ",
"",
"## 汇总",
"",
"| 指标 | 数值 |",
"|------|------|",
f"| 总用例数 | {s['total_cases']} |",
f"| 通过用例 | {s['passed_cases']} |",
f"| 失败用例 | {s['failed_cases']} |",
f"| 总规则数 | {s['total_rules']} |",
f"| 通过规则 | {s['passed_rules']} |",
f"| 通过率 | {s['pass_rate'] * 100:.1f}% |",
f"| 连通用例 | {s.get('connectivity_cases', 0)} |",
f"| 判定型通过率 | {judged_rate_text} |",
"",
"## 用例明细",
"",
]
for case in report.get("cases", []):
if case.get("connectivity"):
badge = "🔗"
else:
badge = "✅" if case.get("passed") else "❌"
title = f"### {badge} 用例 `{case['case_id']}`"
if case.get("connectivity"):
title += "（连通用例，未配置判定标准）"
lines.append(title)
lines.append("")
for turn in case.get("turns", []):
lines.append(f"**第 {turn['round']} 轮**")
lines.append("")
lines.append(f"> **用户**: {turn.get('sent_text', '—')}")
lines.append("")
reply = turn.get("reply_text") or "（无回复）"
lines.append(f"> **智能体**: {reply}")
if turn.get("latency_ms") is not None:
lines.append(f"> *延迟: {turn['latency_ms']}ms*")
lines.append("")
if case.get("results"):
lines.append("**规则评估结果**")
lines.append("")
lines.append("| 规则 | 结果 | 评分 | 说明 |")
lines.append("|------|------|------|------|")
for r in case["results"]:
badge = "✅" if r["passed"] else "❌"
score = f"{r['score']:.2f}" if r.get("score") is not None else "-"
lines.append(f"| {r['rule_type']} | {badge} | {score} | {r.get('reason', '')} |")
lines.append("")
return "\n".join(lines)
def render_campaign_markdown(report: dict[str, Any]) -> str:
"""Render a dual-axis campaign report dict as Markdown."""
s = report["summary"]
lines: list[str] = [
f"# 活动周期报告 — {report['name']}",
"",
f"**状态**: {report['status']} ",
f"**窗口**: {report['window_seconds']}s（倍速 {report['time_scale']}） ",
f"**开始时间**: {report['started_at'] or '-'} ",
f"**完成时间**: {report['completed_at'] or '-'} ",
"",
"## 汇总",
"",
"| 指标 | 数值 |",
"|------|------|",
f"| 子运行总数 | {s['total_runs']} |",
f"| 已完成 | {s['completed_runs']} |",
f"| 整窗通过率 | {_pct(s['overall_pass_rate'])} |",
f"| 整窗可用性 | {_pct(s['overall_availability'])} |",
f"| 平均时延 | {_ms(s['avg_latency_ms'])} |",
"",
"## 时间趋势",
"",
"| 时段(秒) | 运行数 | 通过率 | 可用性 | 时延 |",
"|------|------|------|------|------|",
]
for b in report["time_trend"]:
lines.append(
f"| {b['start_seconds']:.0f}–{b['end_seconds']:.0f} | {b['run_count']} | "
f"{_pct(b['pass_rate'])} | {_pct(b['availability'])} | {_ms(b['avg_latency_ms'])} |"
)
lines += [
"",
"## 能力汇总",
"",
"| 场景 | 运行数 | 通过率 | 可用性 | 时延 |",
"|------|------|------|------|------|",
]
for c in report["capability_summary"]:
lines.append(
f"| {c['scenario_name']} | {c['run_count']} | {_pct(c['pass_rate'])} | "
f"{_pct(c['availability'])} | {_ms(c['avg_latency_ms'])} |"
)
return "\n".join(lines)

9
backend/agenteval/web/routers/campaigns.py
View File

@ -12,11 +12,8 @@ from pydantic import BaseModel, Field
from sqlmodel import Session
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
from agenteval.evaluation.report import (
generate_campaign_report,
render_campaign_markdown_report,
summarize_campaign_progress,
)
from agenteval.evaluation.report import generate_campaign_report, summarize_campaign_progress
from agenteval.evaluation.report_render import render_campaign_markdown
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
from agenteval.storage.db import utc_now
from agenteval.storage.repository import (
@ -114,7 +111,7 @@ async def get_campaign_report_markdown(campaign_id: str, session: Session = Depe
raise HTTPException(status_code=404, detail="campaign not found")
runs = RunRepository(session).list_by_campaign(campaign_id)
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
md = render_campaign_markdown_report(campaign, runs, scenario_names=scenario_names)
md = render_campaign_markdown(generate_campaign_report(campaign, runs, scenario_names=scenario_names))
return Response(
content=md,
media_type="text/markdown; charset=utf-8",

15
backend/agenteval/web/routers/reports.py
View File

@ -3,13 +3,8 @@
from fastapi import APIRouter, Depends, HTTPException, Query, Response
from sqlmodel import Session
from agenteval.evaluation.report import (
generate_compare_report,
generate_report,
render_html_report,
render_json_report,
render_markdown_report,
)
from agenteval.evaluation.report import generate_compare_report, generate_report
from agenteval.evaluation.report_render import render_html, render_json, render_markdown
from agenteval.storage.repository import RunRepository
from agenteval.web.deps import get_db
@ -53,7 +48,7 @@ def get_html_report(run_id: str, session: Session = Depends(get_db)) -> Response
run = RunRepository(session).get(run_id)
if not run:
raise HTTPException(status_code=404, detail="run not found")
html = render_html_report(run_id, session)
html = render_html(generate_report(run_id, session))
return Response(content=html, media_type="text/html")
@ -62,7 +57,7 @@ def get_json_report(run_id: str, session: Session = Depends(get_db)) -> Response
run = RunRepository(session).get(run_id)
if not run:
raise HTTPException(status_code=404, detail="run not found")
json_text = render_json_report(run_id, session)
json_text = render_json(generate_report(run_id, session))
return Response(content=json_text, media_type="application/json")
@ -71,7 +66,7 @@ def get_markdown_report(run_id: str, session: Session = Depends(get_db)) -> Resp
run = RunRepository(session).get(run_id)
if not run:
raise HTTPException(status_code=404, detail="run not found")
md = render_markdown_report(run_id, session)
md = render_markdown(generate_report(run_id, session))
return Response(
content=md,
media_type="text/markdown; charset=utf-8",

14
backend/cli/report.py
View File

@ -4,7 +4,9 @@ from pathlib import Path
from typing import Optional
import typer
from agenteval.evaluation.report import render_html_report, render_json_report, save_report
from agenteval.evaluation.report import generate_report as build_report
from agenteval.evaluation.report import save_report
from agenteval.evaluation.report_render import render_html, render_json
from agenteval.storage.repository import RunRepository
from rich.console import Console
@ -23,9 +25,9 @@ def show_report(
raise typer.Exit(1)
if fmt == "json":
console.print(render_json_report(run_id))
console.print(render_json(build_report(run_id)))
elif fmt == "html":
console.print(render_html_report(run_id))
console.print(render_html(build_report(run_id)))
else:
console.print(f"不支持的格式: {fmt}", style="red")
raise typer.Exit(1)
@ -51,16 +53,14 @@ def compare_reports(
run_id_1: str,
run_id_2: str,
) -> None:
from agenteval.evaluation.report import generate_report
r1 = RunRepository().get(run_id_1)
r2 = RunRepository().get(run_id_2)
if not r1 or not r2:
console.print("评测记录不存在", style="red")
raise typer.Exit(1)
report1 = generate_report(run_id_1)
report2 = generate_report(run_id_2)
report1 = build_report(run_id_1)
report2 = build_report(run_id_2)
console.print(f"对比: {run_id_1} vs {run_id_2}")
console.print(f" 用例数: {report1['summary']['total_cases']} -> {report2['summary']['total_cases']}")

5
backend/cli/run.py
View File

@ -57,9 +57,10 @@ def start_run(
console.print_json(data=run.summary)
if output_json:
from agenteval.evaluation.report import render_json_report
from agenteval.evaluation.report import generate_report
from agenteval.evaluation.report_render import render_json
console.print(render_json_report(run.id))
console.print(render_json(generate_report(run.id)))
@app.command("list", help="列出评测执行记录")

19
tests/unit/test_report.py
View File

@ -6,9 +6,8 @@ from sqlmodel import Session, SQLModel, create_engine
from agenteval.evaluation.report import (
generate_compare_report,
generate_report,
render_markdown_report,
render_json_report,
)
from agenteval.evaluation.report_render import render_json, render_markdown
from agenteval.models import (
Case, CaseType, EvalResult, EvalRun, EvalTarget, RunStatus, Scenario, Turn,
PlatformType, ChannelType, TargetStatus,
@ -318,7 +317,7 @@ def test_compare_report_marks_connectivity(report_session):
def test_markdown_report_shows_connectivity(report_session):
run_id = _seed_run(report_session, n_cases=1, connectivity_cases=1)
md = render_markdown_report(run_id, report_session)
md = render_markdown(generate_report(run_id, report_session))
assert "连通用例" in md
assert "判定型通过率" in md
@ -327,13 +326,13 @@ def test_markdown_report_shows_connectivity(report_session):
def test_render_markdown_contains_header(report_session):
run_id = _seed_run(report_session)
md = render_markdown_report(run_id, report_session)
md = render_markdown(generate_report(run_id, report_session))
assert "# 评测报告" in md
def test_render_markdown_contains_summary_table(report_session):
run_id = _seed_run(report_session)
md = render_markdown_report(run_id, report_session)
md = render_markdown(generate_report(run_id, report_session))
assert "## 汇总" in md
assert "| 指标 | 数值 |" in md
assert "通过率" in md
@ -341,14 +340,14 @@ def test_render_markdown_contains_summary_table(report_session):
def test_render_markdown_contains_case_section(report_session):
run_id = _seed_run(report_session, n_cases=1)
md = render_markdown_report(run_id, report_session)
md = render_markdown(generate_report(run_id, report_session))
assert "## 用例明细" in md
assert "### " in md # case header
def test_render_markdown_contains_rule_table(report_session):
run_id = _seed_run(report_session)
md = render_markdown_report(run_id, report_session)
md = render_markdown(generate_report(run_id, report_session))
assert "**规则评估结果**" in md
assert "keyword_match" in md
@ -358,7 +357,7 @@ def test_render_markdown_contains_rule_table(report_session):
def test_render_json_report_is_valid_json(report_session):
import json
run_id = _seed_run(report_session)
json_text = render_json_report(run_id, report_session)
json_text = render_json(generate_report(run_id, report_session))
parsed = json.loads(json_text)
assert parsed["run_id"] == run_id
@ -380,7 +379,7 @@ def test_case_dict_contains_passed_fallback(report_session):
def test_markdown_errored_case_shows_failed_badge(report_session):
"""故障用例（无结果且非连通）在 MD 中必须 ❌ —— 此前 all([]) 误判 ✅。"""
run_id = _seed_run(report_session, n_cases=1, errored_cases=1)
md = render_markdown_report(run_id, report_session)
md = render_markdown(generate_report(run_id, report_session))
assert "❌ 用例 `err0`" in md
assert "✅ 用例 `err0`" not in md
@ -396,7 +395,7 @@ def test_authoritative_case_outcomes_override_reconstruction(report_session):
report = generate_report(run_id, report_session)
assert report["cases"][0]["passed"] is False
md = render_markdown_report(run_id, report_session)
md = render_markdown(generate_report(run_id, report_session))
assert "❌ 用例 `c0`" in md

187
tests/unit/test_report_render.py Normal file
View File

@ -0,0 +1,187 @@
"""Unit tests for the pure renderers: dict in, HTML/Markdown/JSON out (no DB)."""
import json
from agenteval.evaluation.report_render import (
render_campaign_markdown,
render_html,
render_json,
render_markdown,
)
def _run_report(**overrides) -> dict:
"""A hand-built report dict matching generate_report's shape."""
report = {
"run_id": "run-1",
"target_id": "t-1",
"target_name": "客服机器人",
"scenario_id": "s-1",
"scenario_name": "售后场景",
"scenario_version": 2,
"triggered_by": "manual",
"status": "completed",
"started_at": "2026-07-30T10:00:00+00:00",
"completed_at": "2026-07-30T10:05:00+00:00",
"summary": {
"total_cases": 2,
"passed_cases": 1,
"failed_cases": 1,
"total_rules": 3,
"passed_rules": 2,
"pass_rate": 0.5,
"connectivity_cases": 1,
"judged_pass_rate": 0.0,
},
"cases": [
{
"case_id": "case-a",
"passed": True,
"connectivity": True,
"turns": [
{
"round": 0,
"sent_text": "你好",
"reply_text": "您好，请问有什么可以帮您？",
"latency_ms": 120,
"question_msg_id": "m-1",
}
],
"results": [],
},
{
"case_id": "case-b",
"passed": False,
"connectivity": False,
"turns": [
{
"round": 0,
"sent_text": "退货流程",
"reply_text": None,
"latency_ms": None,
"question_msg_id": "m-2",
}
],
"results": [
{"rule_type": "keyword_match", "passed": False, "score": 0.0, "reason": "缺少关键词"},
],
},
],
}
report.update(overrides)
return report
def _campaign_report() -> dict:
"""A hand-built dict matching generate_campaign_report's shape."""
return {
"campaign_id": "c-1",
"name": "夜间巡检",
"target_id": "t-1",
"status": "completed",
"window_seconds": 7200,
"time_scale": 1.0,
"started_at": "2026-07-30T00:00:00+00:00",
"completed_at": "2026-07-30T02:00:00+00:00",
"summary": {
"total_runs": 2,
"completed_runs": 2,
"overall_pass_rate": 0.75,
"overall_availability": 1.0,
"avg_latency_ms": 150.0,
},
"time_trend": [
{
"bucket_index": 0,
"start_seconds": 0.0,
"end_seconds": 3600.0,
"run_count": 2,
"pass_rate": 0.75,
"availability": 1.0,
"avg_latency_ms": 150.0,
},
{
"bucket_index": 1,
"start_seconds": 3600.0,
"end_seconds": 7200.0,
"run_count": 0,
"pass_rate": None,
"availability": None,
"avg_latency_ms": None,
},
],
"capability_summary": [
{
"scenario_id": "s-1",
"scenario_name": "售后场景",
"run_count": 2,
"pass_rate": 0.75,
"availability": 1.0,
"avg_latency_ms": 150.0,
},
],
}
# ── render_html ─────────────────────────────────────────────────────────────
def test_render_html_contains_names_and_summary():
html = render_html(_run_report())
assert "客服机器人" in html
assert "售后场景" in html
assert "50.00%" in html # pass_rate 0.5
def test_render_html_contains_turns_and_rule_badges():
html = render_html(_run_report())
assert "退货流程" in html
assert "keyword_match" in html
assert "失败" in html
# ── render_markdown ─────────────────────────────────────────────────────────
def test_render_markdown_summary_table():
md = render_markdown(_run_report())
assert "| 总用例数 | 2 |" in md
assert "| 通过率 | 50.0% |" in md
assert "| 连通用例 | 1 |" in md
def test_render_markdown_connectivity_badge_and_no_reply():
md = render_markdown(_run_report())
assert "🔗" in md # connectivity case badge
assert "（连通用例，未配置判定标准）" in md
assert "（无回复）" in md
def test_render_markdown_judged_pass_rate_dash_when_none():
report = _run_report()
report["summary"]["judged_pass_rate"] = None
md = render_markdown(report)
assert "| 判定型通过率 | — |" in md
# ── render_json ─────────────────────────────────────────────────────────────
def test_render_json_roundtrips():
report = _run_report()
parsed = json.loads(render_json(report))
assert parsed == report
# ── render_campaign_markdown ────────────────────────────────────────────────
def test_render_campaign_markdown_summary_and_axes():
md = render_campaign_markdown(_campaign_report())
assert "# 活动周期报告 — 夜间巡检" in md
assert "| 整窗通过率 | 75.0% |" in md
assert "## 时间趋势" in md
assert "## 能力汇总" in md
assert "| 售后场景 | 2 | 75.0% | 100.0% | 150ms |" in md
def test_render_campaign_markdown_empty_bucket_dashes():
md = render_campaign_markdown(_campaign_report())
# bucket 1 has no runs: pass_rate/availability/latency all render as —
assert "| 3600–7200 | 0 | — | — | — |" in md