refactor(report): split report generation from pure rendering
report.py mixed DB-reading generation with string formatting: the four render_*_report(run_id, session) functions each re-fetched via generate_report, so the HTML/Markdown/JSON formatting was welded to storage and could not be unit-tested from a plain dict. Extract the formatting into a new pure report_render module whose renderers take the already-built report dict (no session, no storage import). Migrate every caller to generate-then- render, delete the old coupled renderers with no back-compat shim, and drop the _aggregate_runs middle-man alias in favour of metrics.aggregate_runs.
This commit is contained in:
sinohqb
parent
0cca4963d1
commit
f285738f6d
@ -1,95 +1,21 @@
|
|||||||
"""Report generation for evaluation runs."""
|
"""Report generation for evaluation runs.
|
||||||
|
|
||||||
|
Generation only: read the DB / model objects and build the report dict.
|
||||||
|
Formatting lives in ``report_render`` (pure dict → HTML/Markdown/JSON).
|
||||||
|
"""
|
||||||
|
|
||||||
import json
|
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
from jinja2 import Template
|
|
||||||
|
|
||||||
from agenteval.evaluation.metrics import aggregate_runs
|
from agenteval.evaluation.metrics import aggregate_runs
|
||||||
|
from agenteval.evaluation.report_render import render_html, render_json, render_markdown
|
||||||
from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary
|
from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary
|
||||||
from agenteval.storage.db import DATA_DIR, iso_utc
|
from agenteval.storage.db import DATA_DIR, iso_utc
|
||||||
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
||||||
from agenteval.utils.llm import extract_reply_text
|
from agenteval.utils.llm import extract_reply_text
|
||||||
|
|
||||||
HTML_TEMPLATE = """<!DOCTYPE html>
|
|
||||||
<html lang="zh-CN">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<title>评测报告 - {{ report.run_id }}</title>
|
|
||||||
<style>
|
|
||||||
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; margin: 40px; background: #f5f5f5; }
|
|
||||||
.container { max-width: 960px; margin: 0 auto; background: #fff; padding: 32px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.05); }
|
|
||||||
h1 { margin-top: 0; }
|
|
||||||
.summary { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 16px; margin: 24px 0; }
|
|
||||||
.card { background: #fafafa; border-radius: 6px; padding: 16px; text-align: center; }
|
|
||||||
.card .value { font-size: 24px; font-weight: 700; }
|
|
||||||
.card .label { color: #666; font-size: 14px; margin-top: 4px; }
|
|
||||||
.case { border: 1px solid #e0e0e0; border-radius: 6px; margin: 16px 0; padding: 16px; }
|
|
||||||
.case-title { font-weight: 600; margin-bottom: 8px; }
|
|
||||||
.turn { background: #f9f9f9; border-radius: 4px; padding: 12px; margin: 8px 0; }
|
|
||||||
.message-label { color: #666; font-size: 12px; }
|
|
||||||
.rule { display: flex; align-items: center; gap: 8px; margin: 6px 0; }
|
|
||||||
.badge { padding: 2px 8px; border-radius: 4px; font-size: 12px; }
|
|
||||||
.pass { background: #e6f7e6; color: #2e7d32; }
|
|
||||||
.fail { background: #ffebee; color: #c62828; }
|
|
||||||
pre { white-space: pre-wrap; word-break: break-word; background: #f5f5f5; padding: 8px; border-radius: 4px; }
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<div class="container">
|
|
||||||
<h1>评测报告</h1>
|
|
||||||
<p>评测对象：{{ report.target_name }}（{{ report.target_id }}）</p>
|
|
||||||
<p>评测场景：{{ report.scenario_name }}（{{ report.scenario_id }}）</p>
|
|
||||||
<p>执行时间：{{ report.started_at }} 至 {{ report.completed_at or '进行中' }}</p>
|
|
||||||
|
|
||||||
<div class="summary">
|
|
||||||
<div class="card">
|
|
||||||
<div class="value">{{ report.summary.total_cases }}</div>
|
|
||||||
<div class="label">用例总数</div>
|
|
||||||
</div>
|
|
||||||
<div class="card">
|
|
||||||
<div class="value">{{ report.summary.passed_cases }}</div>
|
|
||||||
<div class="label">通过用例</div>
|
|
||||||
</div>
|
|
||||||
<div class="card">
|
|
||||||
<div class="value">{{ report.summary.total_rules }}</div>
|
|
||||||
<div class="label">规则总数</div>
|
|
||||||
</div>
|
|
||||||
<div class="card">
|
|
||||||
<div class="value">{{ "%.2f"|format(report.summary.pass_rate * 100) }}%</div>
|
|
||||||
<div class="label">规则通过率</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{% for case in report.cases %}
|
|
||||||
<div class="case">
|
|
||||||
<div class="case-title">用例 {{ case.case_id }}</div>
|
|
||||||
{% for turn in case.turns %}
|
|
||||||
<div class="turn">
|
|
||||||
<div class="message-label">用户消息</div>
|
|
||||||
<pre>{{ turn.sent_text }}</pre>
|
|
||||||
<div class="message-label">智能体回复（{{ turn.latency_ms }}ms）</div>
|
|
||||||
<pre>{{ turn.reply_text or '（无回复）' }}</pre>
|
|
||||||
</div>
|
|
||||||
{% endfor %}
|
|
||||||
<div>
|
|
||||||
{% for result in case.results %}
|
|
||||||
<div class="rule">
|
|
||||||
<span class="badge {{ 'pass' if result.passed else 'fail' }}">{{ '通过' if result.passed else '失败' }}</span>
|
|
||||||
<span>{{ result.rule_type }}: {{ result.reason }}</span>
|
|
||||||
</div>
|
|
||||||
{% endfor %}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{% endfor %}
|
|
||||||
</div>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_text(data: Any) -> str:
|
def _extract_text(data: Any) -> str:
|
||||||
return extract_reply_text(data)
|
return extract_reply_text(data)
|
||||||
@ -292,24 +218,19 @@ def _to_utc(dt: Optional[datetime]) -> Optional[datetime]:
|
|||||||
return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt
|
return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt
|
||||||
|
|
||||||
|
|
||||||
def _aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]:
|
|
||||||
"""Delegates to the single cross-run aggregation seam (ADR-0004)."""
|
|
||||||
return aggregate_runs(runs)
|
|
||||||
|
|
||||||
|
|
||||||
def summarize_campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict[str, Any]:
|
def summarize_campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict[str, Any]:
|
||||||
"""Compact list-row progress: completed vs *planned* total, plus pass_rate.
|
"""Compact list-row progress: completed vs *planned* total, plus pass_rate.
|
||||||
|
|
||||||
Unlike ``campaign_progress`` (live window position), this powers the list
|
Unlike ``campaign_progress`` (live window position), this powers the list
|
||||||
view. ``planned_total`` is the sum of plan-entry counts — a fixed target the
|
view. ``planned_total`` is the sum of plan-entry counts — a fixed target the
|
||||||
campaign works toward, so the progress bar fills from 0 rather than tracking
|
campaign works toward, so the progress bar fills from 0 rather than tracking
|
||||||
a growing spawned count. ``overall_pass_rate`` reuses ``_aggregate_runs`` so
|
a growing spawned count. ``overall_pass_rate`` reuses ``aggregate_runs`` so
|
||||||
the list figure matches the report exactly (ADR-0002: failures count as 0.0).
|
the list figure matches the report exactly (ADR-0002: failures count as 0.0).
|
||||||
"""
|
"""
|
||||||
return {
|
return {
|
||||||
"completed_runs": sum(1 for r in runs if r.status == RunStatus.COMPLETED),
|
"completed_runs": sum(1 for r in runs if r.status == RunStatus.COMPLETED),
|
||||||
"planned_total": sum(entry.count for entry in campaign.plan),
|
"planned_total": sum(entry.count for entry in campaign.plan),
|
||||||
"overall_pass_rate": _aggregate_runs(runs)["pass_rate"],
|
"overall_pass_rate": aggregate_runs(runs)["pass_rate"],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -350,7 +271,7 @@ def generate_campaign_report(
|
|||||||
|
|
||||||
time_trend = []
|
time_trend = []
|
||||||
for idx in range(bucket_count):
|
for idx in range(bucket_count):
|
||||||
agg = _aggregate_runs(buckets.get(idx, []))
|
agg = aggregate_runs(buckets.get(idx, []))
|
||||||
time_trend.append({
|
time_trend.append({
|
||||||
"bucket_index": idx,
|
"bucket_index": idx,
|
||||||
"start_seconds": round(idx * bucket_seconds, 3),
|
"start_seconds": round(idx * bucket_seconds, 3),
|
||||||
@ -364,7 +285,7 @@ def generate_campaign_report(
|
|||||||
by_scenario[run.scenario_id].append(run)
|
by_scenario[run.scenario_id].append(run)
|
||||||
capability_summary = []
|
capability_summary = []
|
||||||
for sid, sruns in by_scenario.items():
|
for sid, sruns in by_scenario.items():
|
||||||
agg = _aggregate_runs(sruns)
|
agg = aggregate_runs(sruns)
|
||||||
capability_summary.append({
|
capability_summary.append({
|
||||||
"scenario_id": sid,
|
"scenario_id": sid,
|
||||||
"scenario_name": scenario_names.get(sid, (sid or "")[:8]),
|
"scenario_name": scenario_names.get(sid, (sid or "")[:8]),
|
||||||
@ -372,7 +293,7 @@ def generate_campaign_report(
|
|||||||
})
|
})
|
||||||
capability_summary.sort(key=lambda s: s["run_count"], reverse=True)
|
capability_summary.sort(key=lambda s: s["run_count"], reverse=True)
|
||||||
|
|
||||||
overall = _aggregate_runs(runs)
|
overall = aggregate_runs(runs)
|
||||||
return {
|
return {
|
||||||
"campaign_id": campaign.id,
|
"campaign_id": campaign.id,
|
||||||
"name": campaign.name,
|
"name": campaign.name,
|
||||||
@ -394,163 +315,22 @@ def generate_campaign_report(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def render_campaign_markdown_report(
|
|
||||||
campaign: Campaign,
|
|
||||||
runs: list[EvalRun],
|
|
||||||
*,
|
|
||||||
scenario_names: Optional[dict[str, str]] = None,
|
|
||||||
) -> str:
|
|
||||||
"""Render the dual-axis campaign report as Markdown (reuses the export path)."""
|
|
||||||
report = generate_campaign_report(campaign, runs, scenario_names=scenario_names)
|
|
||||||
s = report["summary"]
|
|
||||||
|
|
||||||
def _pct(v: Optional[float]) -> str:
|
|
||||||
return "—" if v is None else f"{v * 100:.1f}%"
|
|
||||||
|
|
||||||
def _ms(v: Optional[float]) -> str:
|
|
||||||
return "—" if v is None else f"{v:.0f}ms"
|
|
||||||
|
|
||||||
lines: list[str] = [
|
|
||||||
f"# 活动周期报告 — {report['name']}",
|
|
||||||
"",
|
|
||||||
f"**状态**: {report['status']} ",
|
|
||||||
f"**窗口**: {report['window_seconds']}s（倍速 {report['time_scale']}） ",
|
|
||||||
f"**开始时间**: {report['started_at'] or '-'} ",
|
|
||||||
f"**完成时间**: {report['completed_at'] or '-'} ",
|
|
||||||
"",
|
|
||||||
"## 汇总",
|
|
||||||
"",
|
|
||||||
"| 指标 | 数值 |",
|
|
||||||
"|------|------|",
|
|
||||||
f"| 子运行总数 | {s['total_runs']} |",
|
|
||||||
f"| 已完成 | {s['completed_runs']} |",
|
|
||||||
f"| 整窗通过率 | {_pct(s['overall_pass_rate'])} |",
|
|
||||||
f"| 整窗可用性 | {_pct(s['overall_availability'])} |",
|
|
||||||
f"| 平均时延 | {_ms(s['avg_latency_ms'])} |",
|
|
||||||
"",
|
|
||||||
"## 时间趋势",
|
|
||||||
"",
|
|
||||||
"| 时段(秒) | 运行数 | 通过率 | 可用性 | 时延 |",
|
|
||||||
"|------|------|------|------|------|",
|
|
||||||
]
|
|
||||||
for b in report["time_trend"]:
|
|
||||||
lines.append(
|
|
||||||
f"| {b['start_seconds']:.0f}–{b['end_seconds']:.0f} | {b['run_count']} | "
|
|
||||||
f"{_pct(b['pass_rate'])} | {_pct(b['availability'])} | {_ms(b['avg_latency_ms'])} |"
|
|
||||||
)
|
|
||||||
lines += [
|
|
||||||
"",
|
|
||||||
"## 能力汇总",
|
|
||||||
"",
|
|
||||||
"| 场景 | 运行数 | 通过率 | 可用性 | 时延 |",
|
|
||||||
"|------|------|------|------|------|",
|
|
||||||
]
|
|
||||||
for c in report["capability_summary"]:
|
|
||||||
lines.append(
|
|
||||||
f"| {c['scenario_name']} | {c['run_count']} | {_pct(c['pass_rate'])} | "
|
|
||||||
f"{_pct(c['availability'])} | {_ms(c['avg_latency_ms'])} |"
|
|
||||||
)
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
|
|
||||||
def render_markdown_report(run_id: str, session=None) -> str:
|
|
||||||
"""Render a report as Markdown string."""
|
|
||||||
report = generate_report(run_id, session)
|
|
||||||
s = report["summary"]
|
|
||||||
judged_rate = s.get("judged_pass_rate")
|
|
||||||
judged_rate_text = "—" if judged_rate is None else f"{judged_rate * 100:.1f}%"
|
|
||||||
lines: list[str] = [
|
|
||||||
f"# 评测报告 — {report.get('scenario_name', run_id)}",
|
|
||||||
"",
|
|
||||||
f"**评测对象**: {report.get('target_name', '-')} ",
|
|
||||||
f"**评测场景**: {report.get('scenario_name', '-')} ",
|
|
||||||
f"**状态**: {report.get('status', '-')} ",
|
|
||||||
f"**开始时间**: {report.get('started_at', '-')} ",
|
|
||||||
f"**完成时间**: {report.get('completed_at', '-')} ",
|
|
||||||
"",
|
|
||||||
"## 汇总",
|
|
||||||
"",
|
|
||||||
"| 指标 | 数值 |",
|
|
||||||
"|------|------|",
|
|
||||||
f"| 总用例数 | {s['total_cases']} |",
|
|
||||||
f"| 通过用例 | {s['passed_cases']} |",
|
|
||||||
f"| 失败用例 | {s['failed_cases']} |",
|
|
||||||
f"| 总规则数 | {s['total_rules']} |",
|
|
||||||
f"| 通过规则 | {s['passed_rules']} |",
|
|
||||||
f"| 通过率 | {s['pass_rate'] * 100:.1f}% |",
|
|
||||||
f"| 连通用例 | {s.get('connectivity_cases', 0)} |",
|
|
||||||
f"| 判定型通过率 | {judged_rate_text} |",
|
|
||||||
"",
|
|
||||||
"## 用例明细",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
|
|
||||||
for case in report.get("cases", []):
|
|
||||||
if case.get("connectivity"):
|
|
||||||
badge = "🔗"
|
|
||||||
else:
|
|
||||||
badge = "✅" if case.get("passed") else "❌"
|
|
||||||
title = f"### {badge} 用例 `{case['case_id']}`"
|
|
||||||
if case.get("connectivity"):
|
|
||||||
title += "（连通用例，未配置判定标准）"
|
|
||||||
lines.append(title)
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
for turn in case.get("turns", []):
|
|
||||||
lines.append(f"**第 {turn['round']} 轮**")
|
|
||||||
lines.append("")
|
|
||||||
lines.append(f"> **用户**: {turn.get('sent_text', '—')}")
|
|
||||||
lines.append("")
|
|
||||||
reply = turn.get("reply_text") or "（无回复）"
|
|
||||||
lines.append(f"> **智能体**: {reply}")
|
|
||||||
if turn.get("latency_ms") is not None:
|
|
||||||
lines.append(f"> *延迟: {turn['latency_ms']}ms*")
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
if case.get("results"):
|
|
||||||
lines.append("**规则评估结果**")
|
|
||||||
lines.append("")
|
|
||||||
lines.append("| 规则 | 结果 | 评分 | 说明 |")
|
|
||||||
lines.append("|------|------|------|------|")
|
|
||||||
for r in case["results"]:
|
|
||||||
badge = "✅" if r["passed"] else "❌"
|
|
||||||
score = f"{r['score']:.2f}" if r.get("score") is not None else "-"
|
|
||||||
lines.append(f"| {r['rule_type']} | {badge} | {score} | {r.get('reason', '')} |")
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
|
|
||||||
def render_json_report(run_id: str, session=None) -> str:
|
|
||||||
"""Render a report as JSON string."""
|
|
||||||
report = generate_report(run_id, session)
|
|
||||||
return json.dumps(report, ensure_ascii=False, indent=2)
|
|
||||||
|
|
||||||
|
|
||||||
def render_html_report(run_id: str, session=None) -> str:
|
|
||||||
"""Render a report as HTML string."""
|
|
||||||
report = generate_report(run_id, session)
|
|
||||||
template = Template(HTML_TEMPLATE)
|
|
||||||
return template.render(report=report)
|
|
||||||
|
|
||||||
|
|
||||||
def save_report(run_id: str, fmt: str = "html", output_dir: Optional[Path] = None) -> Path:
|
def save_report(run_id: str, fmt: str = "html", output_dir: Optional[Path] = None) -> Path:
|
||||||
"""Generate and save a report to disk."""
|
"""Generate a run report and save it to disk in the requested format."""
|
||||||
output_dir = output_dir or DATA_DIR / "reports"
|
output_dir = output_dir or DATA_DIR / "reports"
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
report = generate_report(run_id)
|
||||||
if fmt == "html":
|
renderers = {
|
||||||
content = render_html_report(run_id)
|
"html": (render_html, "html"),
|
||||||
path = output_dir / f"report_{run_id}_{timestamp}.html"
|
"json": (render_json, "json"),
|
||||||
elif fmt == "json":
|
"markdown": (render_markdown, "md"),
|
||||||
content = render_json_report(run_id)
|
}
|
||||||
path = output_dir / f"report_{run_id}_{timestamp}.json"
|
if fmt not in renderers:
|
||||||
elif fmt == "markdown":
|
|
||||||
content = render_markdown_report(run_id)
|
|
||||||
path = output_dir / f"report_{run_id}_{timestamp}.md"
|
|
||||||
else:
|
|
||||||
raise ValueError(f"unsupported report format: {fmt}")
|
raise ValueError(f"unsupported report format: {fmt}")
|
||||||
|
render, ext = renderers[fmt]
|
||||||
|
|
||||||
path.write_text(content, encoding="utf-8")
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||||
|
path = output_dir / f"report_{run_id}_{timestamp}.{ext}"
|
||||||
|
path.write_text(render(report), encoding="utf-8")
|
||||||
return path
|
return path
|
||||||
|
|||||||
219
backend/agenteval/evaluation/report_render.py
Normal file
219
backend/agenteval/evaluation/report_render.py
Normal file
@ -0,0 +1,219 @@
|
|||||||
|
"""Pure renderers: report dict in, HTML/Markdown/JSON string out.
|
||||||
|
|
||||||
|
No I/O and no storage access — every function takes the dict produced by the
|
||||||
|
generation side (``evaluation.report``) so rendering can be unit-tested from a
|
||||||
|
hand-built dict. Report *content* decisions (which cases pass, which rates to
|
||||||
|
show) belong to generation; this module only formats.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from jinja2 import Template
|
||||||
|
|
||||||
|
HTML_TEMPLATE = """<!DOCTYPE html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<title>评测报告 - {{ report.run_id }}</title>
|
||||||
|
<style>
|
||||||
|
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; margin: 40px; background: #f5f5f5; }
|
||||||
|
.container { max-width: 960px; margin: 0 auto; background: #fff; padding: 32px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.05); }
|
||||||
|
h1 { margin-top: 0; }
|
||||||
|
.summary { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 16px; margin: 24px 0; }
|
||||||
|
.card { background: #fafafa; border-radius: 6px; padding: 16px; text-align: center; }
|
||||||
|
.card .value { font-size: 24px; font-weight: 700; }
|
||||||
|
.card .label { color: #666; font-size: 14px; margin-top: 4px; }
|
||||||
|
.case { border: 1px solid #e0e0e0; border-radius: 6px; margin: 16px 0; padding: 16px; }
|
||||||
|
.case-title { font-weight: 600; margin-bottom: 8px; }
|
||||||
|
.turn { background: #f9f9f9; border-radius: 4px; padding: 12px; margin: 8px 0; }
|
||||||
|
.message-label { color: #666; font-size: 12px; }
|
||||||
|
.rule { display: flex; align-items: center; gap: 8px; margin: 6px 0; }
|
||||||
|
.badge { padding: 2px 8px; border-radius: 4px; font-size: 12px; }
|
||||||
|
.pass { background: #e6f7e6; color: #2e7d32; }
|
||||||
|
.fail { background: #ffebee; color: #c62828; }
|
||||||
|
pre { white-space: pre-wrap; word-break: break-word; background: #f5f5f5; padding: 8px; border-radius: 4px; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="container">
|
||||||
|
<h1>评测报告</h1>
|
||||||
|
<p>评测对象：{{ report.target_name }}（{{ report.target_id }}）</p>
|
||||||
|
<p>评测场景：{{ report.scenario_name }}（{{ report.scenario_id }}）</p>
|
||||||
|
<p>执行时间：{{ report.started_at }} 至 {{ report.completed_at or '进行中' }}</p>
|
||||||
|
|
||||||
|
<div class="summary">
|
||||||
|
<div class="card">
|
||||||
|
<div class="value">{{ report.summary.total_cases }}</div>
|
||||||
|
<div class="label">用例总数</div>
|
||||||
|
</div>
|
||||||
|
<div class="card">
|
||||||
|
<div class="value">{{ report.summary.passed_cases }}</div>
|
||||||
|
<div class="label">通过用例</div>
|
||||||
|
</div>
|
||||||
|
<div class="card">
|
||||||
|
<div class="value">{{ report.summary.total_rules }}</div>
|
||||||
|
<div class="label">规则总数</div>
|
||||||
|
</div>
|
||||||
|
<div class="card">
|
||||||
|
<div class="value">{{ "%.2f"|format(report.summary.pass_rate * 100) }}%</div>
|
||||||
|
<div class="label">规则通过率</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% for case in report.cases %}
|
||||||
|
<div class="case">
|
||||||
|
<div class="case-title">用例 {{ case.case_id }}</div>
|
||||||
|
{% for turn in case.turns %}
|
||||||
|
<div class="turn">
|
||||||
|
<div class="message-label">用户消息</div>
|
||||||
|
<pre>{{ turn.sent_text }}</pre>
|
||||||
|
<div class="message-label">智能体回复（{{ turn.latency_ms }}ms）</div>
|
||||||
|
<pre>{{ turn.reply_text or '（无回复）' }}</pre>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
<div>
|
||||||
|
{% for result in case.results %}
|
||||||
|
<div class="rule">
|
||||||
|
<span class="badge {{ 'pass' if result.passed else 'fail' }}">{{ '通过' if result.passed else '失败' }}</span>
|
||||||
|
<span>{{ result.rule_type }}: {{ result.reason }}</span>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _pct(v: Optional[float]) -> str:
|
||||||
|
return "—" if v is None else f"{v * 100:.1f}%"
|
||||||
|
|
||||||
|
|
||||||
|
def _ms(v: Optional[float]) -> str:
|
||||||
|
return "—" if v is None else f"{v:.0f}ms"
|
||||||
|
|
||||||
|
|
||||||
|
def render_html(report: dict[str, Any]) -> str:
|
||||||
|
"""Render a run report dict as an HTML string."""
|
||||||
|
return Template(HTML_TEMPLATE).render(report=report)
|
||||||
|
|
||||||
|
|
||||||
|
def render_json(report: dict[str, Any]) -> str:
|
||||||
|
"""Render a report dict as a JSON string."""
|
||||||
|
return json.dumps(report, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
|
||||||
|
def render_markdown(report: dict[str, Any]) -> str:
|
||||||
|
"""Render a run report dict as Markdown."""
|
||||||
|
s = report["summary"]
|
||||||
|
judged_rate = s.get("judged_pass_rate")
|
||||||
|
judged_rate_text = "—" if judged_rate is None else f"{judged_rate * 100:.1f}%"
|
||||||
|
lines: list[str] = [
|
||||||
|
f"# 评测报告 — {report.get('scenario_name', report.get('run_id', ''))}",
|
||||||
|
"",
|
||||||
|
f"**评测对象**: {report.get('target_name', '-')} ",
|
||||||
|
f"**评测场景**: {report.get('scenario_name', '-')} ",
|
||||||
|
f"**状态**: {report.get('status', '-')} ",
|
||||||
|
f"**开始时间**: {report.get('started_at', '-')} ",
|
||||||
|
f"**完成时间**: {report.get('completed_at', '-')} ",
|
||||||
|
"",
|
||||||
|
"## 汇总",
|
||||||
|
"",
|
||||||
|
"| 指标 | 数值 |",
|
||||||
|
"|------|------|",
|
||||||
|
f"| 总用例数 | {s['total_cases']} |",
|
||||||
|
f"| 通过用例 | {s['passed_cases']} |",
|
||||||
|
f"| 失败用例 | {s['failed_cases']} |",
|
||||||
|
f"| 总规则数 | {s['total_rules']} |",
|
||||||
|
f"| 通过规则 | {s['passed_rules']} |",
|
||||||
|
f"| 通过率 | {s['pass_rate'] * 100:.1f}% |",
|
||||||
|
f"| 连通用例 | {s.get('connectivity_cases', 0)} |",
|
||||||
|
f"| 判定型通过率 | {judged_rate_text} |",
|
||||||
|
"",
|
||||||
|
"## 用例明细",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
for case in report.get("cases", []):
|
||||||
|
if case.get("connectivity"):
|
||||||
|
badge = "🔗"
|
||||||
|
else:
|
||||||
|
badge = "✅" if case.get("passed") else "❌"
|
||||||
|
title = f"### {badge} 用例 `{case['case_id']}`"
|
||||||
|
if case.get("connectivity"):
|
||||||
|
title += "（连通用例，未配置判定标准）"
|
||||||
|
lines.append(title)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
for turn in case.get("turns", []):
|
||||||
|
lines.append(f"**第 {turn['round']} 轮**")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(f"> **用户**: {turn.get('sent_text', '—')}")
|
||||||
|
lines.append("")
|
||||||
|
reply = turn.get("reply_text") or "（无回复）"
|
||||||
|
lines.append(f"> **智能体**: {reply}")
|
||||||
|
if turn.get("latency_ms") is not None:
|
||||||
|
lines.append(f"> *延迟: {turn['latency_ms']}ms*")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
if case.get("results"):
|
||||||
|
lines.append("**规则评估结果**")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("| 规则 | 结果 | 评分 | 说明 |")
|
||||||
|
lines.append("|------|------|------|------|")
|
||||||
|
for r in case["results"]:
|
||||||
|
badge = "✅" if r["passed"] else "❌"
|
||||||
|
score = f"{r['score']:.2f}" if r.get("score") is not None else "-"
|
||||||
|
lines.append(f"| {r['rule_type']} | {badge} | {score} | {r.get('reason', '')} |")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def render_campaign_markdown(report: dict[str, Any]) -> str:
|
||||||
|
"""Render a dual-axis campaign report dict as Markdown."""
|
||||||
|
s = report["summary"]
|
||||||
|
lines: list[str] = [
|
||||||
|
f"# 活动周期报告 — {report['name']}",
|
||||||
|
"",
|
||||||
|
f"**状态**: {report['status']} ",
|
||||||
|
f"**窗口**: {report['window_seconds']}s（倍速 {report['time_scale']}） ",
|
||||||
|
f"**开始时间**: {report['started_at'] or '-'} ",
|
||||||
|
f"**完成时间**: {report['completed_at'] or '-'} ",
|
||||||
|
"",
|
||||||
|
"## 汇总",
|
||||||
|
"",
|
||||||
|
"| 指标 | 数值 |",
|
||||||
|
"|------|------|",
|
||||||
|
f"| 子运行总数 | {s['total_runs']} |",
|
||||||
|
f"| 已完成 | {s['completed_runs']} |",
|
||||||
|
f"| 整窗通过率 | {_pct(s['overall_pass_rate'])} |",
|
||||||
|
f"| 整窗可用性 | {_pct(s['overall_availability'])} |",
|
||||||
|
f"| 平均时延 | {_ms(s['avg_latency_ms'])} |",
|
||||||
|
"",
|
||||||
|
"## 时间趋势",
|
||||||
|
"",
|
||||||
|
"| 时段(秒) | 运行数 | 通过率 | 可用性 | 时延 |",
|
||||||
|
"|------|------|------|------|------|",
|
||||||
|
]
|
||||||
|
for b in report["time_trend"]:
|
||||||
|
lines.append(
|
||||||
|
f"| {b['start_seconds']:.0f}–{b['end_seconds']:.0f} | {b['run_count']} | "
|
||||||
|
f"{_pct(b['pass_rate'])} | {_pct(b['availability'])} | {_ms(b['avg_latency_ms'])} |"
|
||||||
|
)
|
||||||
|
lines += [
|
||||||
|
"",
|
||||||
|
"## 能力汇总",
|
||||||
|
"",
|
||||||
|
"| 场景 | 运行数 | 通过率 | 可用性 | 时延 |",
|
||||||
|
"|------|------|------|------|------|",
|
||||||
|
]
|
||||||
|
for c in report["capability_summary"]:
|
||||||
|
lines.append(
|
||||||
|
f"| {c['scenario_name']} | {c['run_count']} | {_pct(c['pass_rate'])} | "
|
||||||
|
f"{_pct(c['availability'])} | {_ms(c['avg_latency_ms'])} |"
|
||||||
|
)
|
||||||
|
return "\n".join(lines)
|
||||||
@ -12,11 +12,8 @@ from pydantic import BaseModel, Field
|
|||||||
from sqlmodel import Session
|
from sqlmodel import Session
|
||||||
|
|
||||||
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
||||||
from agenteval.evaluation.report import (
|
from agenteval.evaluation.report import generate_campaign_report, summarize_campaign_progress
|
||||||
generate_campaign_report,
|
from agenteval.evaluation.report_render import render_campaign_markdown
|
||||||
render_campaign_markdown_report,
|
|
||||||
summarize_campaign_progress,
|
|
||||||
)
|
|
||||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
|
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
|
||||||
from agenteval.storage.db import utc_now
|
from agenteval.storage.db import utc_now
|
||||||
from agenteval.storage.repository import (
|
from agenteval.storage.repository import (
|
||||||
@ -114,7 +111,7 @@ async def get_campaign_report_markdown(campaign_id: str, session: Session = Depe
|
|||||||
raise HTTPException(status_code=404, detail="campaign not found")
|
raise HTTPException(status_code=404, detail="campaign not found")
|
||||||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||||||
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
||||||
md = render_campaign_markdown_report(campaign, runs, scenario_names=scenario_names)
|
md = render_campaign_markdown(generate_campaign_report(campaign, runs, scenario_names=scenario_names))
|
||||||
return Response(
|
return Response(
|
||||||
content=md,
|
content=md,
|
||||||
media_type="text/markdown; charset=utf-8",
|
media_type="text/markdown; charset=utf-8",
|
||||||
|
|||||||
@ -3,13 +3,8 @@
|
|||||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response
|
from fastapi import APIRouter, Depends, HTTPException, Query, Response
|
||||||
from sqlmodel import Session
|
from sqlmodel import Session
|
||||||
|
|
||||||
from agenteval.evaluation.report import (
|
from agenteval.evaluation.report import generate_compare_report, generate_report
|
||||||
generate_compare_report,
|
from agenteval.evaluation.report_render import render_html, render_json, render_markdown
|
||||||
generate_report,
|
|
||||||
render_html_report,
|
|
||||||
render_json_report,
|
|
||||||
render_markdown_report,
|
|
||||||
)
|
|
||||||
from agenteval.storage.repository import RunRepository
|
from agenteval.storage.repository import RunRepository
|
||||||
from agenteval.web.deps import get_db
|
from agenteval.web.deps import get_db
|
||||||
|
|
||||||
@ -53,7 +48,7 @@ def get_html_report(run_id: str, session: Session = Depends(get_db)) -> Response
|
|||||||
run = RunRepository(session).get(run_id)
|
run = RunRepository(session).get(run_id)
|
||||||
if not run:
|
if not run:
|
||||||
raise HTTPException(status_code=404, detail="run not found")
|
raise HTTPException(status_code=404, detail="run not found")
|
||||||
html = render_html_report(run_id, session)
|
html = render_html(generate_report(run_id, session))
|
||||||
return Response(content=html, media_type="text/html")
|
return Response(content=html, media_type="text/html")
|
||||||
|
|
||||||
|
|
||||||
@ -62,7 +57,7 @@ def get_json_report(run_id: str, session: Session = Depends(get_db)) -> Response
|
|||||||
run = RunRepository(session).get(run_id)
|
run = RunRepository(session).get(run_id)
|
||||||
if not run:
|
if not run:
|
||||||
raise HTTPException(status_code=404, detail="run not found")
|
raise HTTPException(status_code=404, detail="run not found")
|
||||||
json_text = render_json_report(run_id, session)
|
json_text = render_json(generate_report(run_id, session))
|
||||||
return Response(content=json_text, media_type="application/json")
|
return Response(content=json_text, media_type="application/json")
|
||||||
|
|
||||||
|
|
||||||
@ -71,7 +66,7 @@ def get_markdown_report(run_id: str, session: Session = Depends(get_db)) -> Resp
|
|||||||
run = RunRepository(session).get(run_id)
|
run = RunRepository(session).get(run_id)
|
||||||
if not run:
|
if not run:
|
||||||
raise HTTPException(status_code=404, detail="run not found")
|
raise HTTPException(status_code=404, detail="run not found")
|
||||||
md = render_markdown_report(run_id, session)
|
md = render_markdown(generate_report(run_id, session))
|
||||||
return Response(
|
return Response(
|
||||||
content=md,
|
content=md,
|
||||||
media_type="text/markdown; charset=utf-8",
|
media_type="text/markdown; charset=utf-8",
|
||||||
|
|||||||
@ -4,7 +4,9 @@ from pathlib import Path
|
|||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import typer
|
import typer
|
||||||
from agenteval.evaluation.report import render_html_report, render_json_report, save_report
|
from agenteval.evaluation.report import generate_report as build_report
|
||||||
|
from agenteval.evaluation.report import save_report
|
||||||
|
from agenteval.evaluation.report_render import render_html, render_json
|
||||||
from agenteval.storage.repository import RunRepository
|
from agenteval.storage.repository import RunRepository
|
||||||
from rich.console import Console
|
from rich.console import Console
|
||||||
|
|
||||||
@ -23,9 +25,9 @@ def show_report(
|
|||||||
raise typer.Exit(1)
|
raise typer.Exit(1)
|
||||||
|
|
||||||
if fmt == "json":
|
if fmt == "json":
|
||||||
console.print(render_json_report(run_id))
|
console.print(render_json(build_report(run_id)))
|
||||||
elif fmt == "html":
|
elif fmt == "html":
|
||||||
console.print(render_html_report(run_id))
|
console.print(render_html(build_report(run_id)))
|
||||||
else:
|
else:
|
||||||
console.print(f"不支持的格式: {fmt}", style="red")
|
console.print(f"不支持的格式: {fmt}", style="red")
|
||||||
raise typer.Exit(1)
|
raise typer.Exit(1)
|
||||||
@ -51,16 +53,14 @@ def compare_reports(
|
|||||||
run_id_1: str,
|
run_id_1: str,
|
||||||
run_id_2: str,
|
run_id_2: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
from agenteval.evaluation.report import generate_report
|
|
||||||
|
|
||||||
r1 = RunRepository().get(run_id_1)
|
r1 = RunRepository().get(run_id_1)
|
||||||
r2 = RunRepository().get(run_id_2)
|
r2 = RunRepository().get(run_id_2)
|
||||||
if not r1 or not r2:
|
if not r1 or not r2:
|
||||||
console.print("评测记录不存在", style="red")
|
console.print("评测记录不存在", style="red")
|
||||||
raise typer.Exit(1)
|
raise typer.Exit(1)
|
||||||
|
|
||||||
report1 = generate_report(run_id_1)
|
report1 = build_report(run_id_1)
|
||||||
report2 = generate_report(run_id_2)
|
report2 = build_report(run_id_2)
|
||||||
|
|
||||||
console.print(f"对比: {run_id_1} vs {run_id_2}")
|
console.print(f"对比: {run_id_1} vs {run_id_2}")
|
||||||
console.print(f" 用例数: {report1['summary']['total_cases']} -> {report2['summary']['total_cases']}")
|
console.print(f" 用例数: {report1['summary']['total_cases']} -> {report2['summary']['total_cases']}")
|
||||||
|
|||||||
@ -57,9 +57,10 @@ def start_run(
|
|||||||
console.print_json(data=run.summary)
|
console.print_json(data=run.summary)
|
||||||
|
|
||||||
if output_json:
|
if output_json:
|
||||||
from agenteval.evaluation.report import render_json_report
|
from agenteval.evaluation.report import generate_report
|
||||||
|
from agenteval.evaluation.report_render import render_json
|
||||||
|
|
||||||
console.print(render_json_report(run.id))
|
console.print(render_json(generate_report(run.id)))
|
||||||
|
|
||||||
|
|
||||||
@app.command("list", help="列出评测执行记录")
|
@app.command("list", help="列出评测执行记录")
|
||||||
|
|||||||
@ -6,9 +6,8 @@ from sqlmodel import Session, SQLModel, create_engine
|
|||||||
from agenteval.evaluation.report import (
|
from agenteval.evaluation.report import (
|
||||||
generate_compare_report,
|
generate_compare_report,
|
||||||
generate_report,
|
generate_report,
|
||||||
render_markdown_report,
|
|
||||||
render_json_report,
|
|
||||||
)
|
)
|
||||||
|
from agenteval.evaluation.report_render import render_json, render_markdown
|
||||||
from agenteval.models import (
|
from agenteval.models import (
|
||||||
Case, CaseType, EvalResult, EvalRun, EvalTarget, RunStatus, Scenario, Turn,
|
Case, CaseType, EvalResult, EvalRun, EvalTarget, RunStatus, Scenario, Turn,
|
||||||
PlatformType, ChannelType, TargetStatus,
|
PlatformType, ChannelType, TargetStatus,
|
||||||
@ -318,7 +317,7 @@ def test_compare_report_marks_connectivity(report_session):
|
|||||||
|
|
||||||
def test_markdown_report_shows_connectivity(report_session):
|
def test_markdown_report_shows_connectivity(report_session):
|
||||||
run_id = _seed_run(report_session, n_cases=1, connectivity_cases=1)
|
run_id = _seed_run(report_session, n_cases=1, connectivity_cases=1)
|
||||||
md = render_markdown_report(run_id, report_session)
|
md = render_markdown(generate_report(run_id, report_session))
|
||||||
assert "连通用例" in md
|
assert "连通用例" in md
|
||||||
assert "判定型通过率" in md
|
assert "判定型通过率" in md
|
||||||
|
|
||||||
@ -327,13 +326,13 @@ def test_markdown_report_shows_connectivity(report_session):
|
|||||||
|
|
||||||
def test_render_markdown_contains_header(report_session):
|
def test_render_markdown_contains_header(report_session):
|
||||||
run_id = _seed_run(report_session)
|
run_id = _seed_run(report_session)
|
||||||
md = render_markdown_report(run_id, report_session)
|
md = render_markdown(generate_report(run_id, report_session))
|
||||||
assert "# 评测报告" in md
|
assert "# 评测报告" in md
|
||||||
|
|
||||||
|
|
||||||
def test_render_markdown_contains_summary_table(report_session):
|
def test_render_markdown_contains_summary_table(report_session):
|
||||||
run_id = _seed_run(report_session)
|
run_id = _seed_run(report_session)
|
||||||
md = render_markdown_report(run_id, report_session)
|
md = render_markdown(generate_report(run_id, report_session))
|
||||||
assert "## 汇总" in md
|
assert "## 汇总" in md
|
||||||
assert "| 指标 | 数值 |" in md
|
assert "| 指标 | 数值 |" in md
|
||||||
assert "通过率" in md
|
assert "通过率" in md
|
||||||
@ -341,14 +340,14 @@ def test_render_markdown_contains_summary_table(report_session):
|
|||||||
|
|
||||||
def test_render_markdown_contains_case_section(report_session):
|
def test_render_markdown_contains_case_section(report_session):
|
||||||
run_id = _seed_run(report_session, n_cases=1)
|
run_id = _seed_run(report_session, n_cases=1)
|
||||||
md = render_markdown_report(run_id, report_session)
|
md = render_markdown(generate_report(run_id, report_session))
|
||||||
assert "## 用例明细" in md
|
assert "## 用例明细" in md
|
||||||
assert "### " in md # case header
|
assert "### " in md # case header
|
||||||
|
|
||||||
|
|
||||||
def test_render_markdown_contains_rule_table(report_session):
|
def test_render_markdown_contains_rule_table(report_session):
|
||||||
run_id = _seed_run(report_session)
|
run_id = _seed_run(report_session)
|
||||||
md = render_markdown_report(run_id, report_session)
|
md = render_markdown(generate_report(run_id, report_session))
|
||||||
assert "**规则评估结果**" in md
|
assert "**规则评估结果**" in md
|
||||||
assert "keyword_match" in md
|
assert "keyword_match" in md
|
||||||
|
|
||||||
@ -358,7 +357,7 @@ def test_render_markdown_contains_rule_table(report_session):
|
|||||||
def test_render_json_report_is_valid_json(report_session):
|
def test_render_json_report_is_valid_json(report_session):
|
||||||
import json
|
import json
|
||||||
run_id = _seed_run(report_session)
|
run_id = _seed_run(report_session)
|
||||||
json_text = render_json_report(run_id, report_session)
|
json_text = render_json(generate_report(run_id, report_session))
|
||||||
parsed = json.loads(json_text)
|
parsed = json.loads(json_text)
|
||||||
assert parsed["run_id"] == run_id
|
assert parsed["run_id"] == run_id
|
||||||
|
|
||||||
@ -380,7 +379,7 @@ def test_case_dict_contains_passed_fallback(report_session):
|
|||||||
def test_markdown_errored_case_shows_failed_badge(report_session):
|
def test_markdown_errored_case_shows_failed_badge(report_session):
|
||||||
"""故障用例（无结果且非连通）在 MD 中必须 ❌ —— 此前 all([]) 误判 ✅。"""
|
"""故障用例（无结果且非连通）在 MD 中必须 ❌ —— 此前 all([]) 误判 ✅。"""
|
||||||
run_id = _seed_run(report_session, n_cases=1, errored_cases=1)
|
run_id = _seed_run(report_session, n_cases=1, errored_cases=1)
|
||||||
md = render_markdown_report(run_id, report_session)
|
md = render_markdown(generate_report(run_id, report_session))
|
||||||
assert "❌ 用例 `err0`" in md
|
assert "❌ 用例 `err0`" in md
|
||||||
assert "✅ 用例 `err0`" not in md
|
assert "✅ 用例 `err0`" not in md
|
||||||
|
|
||||||
@ -396,7 +395,7 @@ def test_authoritative_case_outcomes_override_reconstruction(report_session):
|
|||||||
|
|
||||||
report = generate_report(run_id, report_session)
|
report = generate_report(run_id, report_session)
|
||||||
assert report["cases"][0]["passed"] is False
|
assert report["cases"][0]["passed"] is False
|
||||||
md = render_markdown_report(run_id, report_session)
|
md = render_markdown(generate_report(run_id, report_session))
|
||||||
assert "❌ 用例 `c0`" in md
|
assert "❌ 用例 `c0`" in md
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
187
tests/unit/test_report_render.py
Normal file
187
tests/unit/test_report_render.py
Normal file
@ -0,0 +1,187 @@
|
|||||||
|
"""Unit tests for the pure renderers: dict in, HTML/Markdown/JSON out (no DB)."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
from agenteval.evaluation.report_render import (
|
||||||
|
render_campaign_markdown,
|
||||||
|
render_html,
|
||||||
|
render_json,
|
||||||
|
render_markdown,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_report(**overrides) -> dict:
|
||||||
|
"""A hand-built report dict matching generate_report's shape."""
|
||||||
|
report = {
|
||||||
|
"run_id": "run-1",
|
||||||
|
"target_id": "t-1",
|
||||||
|
"target_name": "客服机器人",
|
||||||
|
"scenario_id": "s-1",
|
||||||
|
"scenario_name": "售后场景",
|
||||||
|
"scenario_version": 2,
|
||||||
|
"triggered_by": "manual",
|
||||||
|
"status": "completed",
|
||||||
|
"started_at": "2026-07-30T10:00:00+00:00",
|
||||||
|
"completed_at": "2026-07-30T10:05:00+00:00",
|
||||||
|
"summary": {
|
||||||
|
"total_cases": 2,
|
||||||
|
"passed_cases": 1,
|
||||||
|
"failed_cases": 1,
|
||||||
|
"total_rules": 3,
|
||||||
|
"passed_rules": 2,
|
||||||
|
"pass_rate": 0.5,
|
||||||
|
"connectivity_cases": 1,
|
||||||
|
"judged_pass_rate": 0.0,
|
||||||
|
},
|
||||||
|
"cases": [
|
||||||
|
{
|
||||||
|
"case_id": "case-a",
|
||||||
|
"passed": True,
|
||||||
|
"connectivity": True,
|
||||||
|
"turns": [
|
||||||
|
{
|
||||||
|
"round": 0,
|
||||||
|
"sent_text": "你好",
|
||||||
|
"reply_text": "您好，请问有什么可以帮您？",
|
||||||
|
"latency_ms": 120,
|
||||||
|
"question_msg_id": "m-1",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"results": [],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"case_id": "case-b",
|
||||||
|
"passed": False,
|
||||||
|
"connectivity": False,
|
||||||
|
"turns": [
|
||||||
|
{
|
||||||
|
"round": 0,
|
||||||
|
"sent_text": "退货流程",
|
||||||
|
"reply_text": None,
|
||||||
|
"latency_ms": None,
|
||||||
|
"question_msg_id": "m-2",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"results": [
|
||||||
|
{"rule_type": "keyword_match", "passed": False, "score": 0.0, "reason": "缺少关键词"},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
report.update(overrides)
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def _campaign_report() -> dict:
|
||||||
|
"""A hand-built dict matching generate_campaign_report's shape."""
|
||||||
|
return {
|
||||||
|
"campaign_id": "c-1",
|
||||||
|
"name": "夜间巡检",
|
||||||
|
"target_id": "t-1",
|
||||||
|
"status": "completed",
|
||||||
|
"window_seconds": 7200,
|
||||||
|
"time_scale": 1.0,
|
||||||
|
"started_at": "2026-07-30T00:00:00+00:00",
|
||||||
|
"completed_at": "2026-07-30T02:00:00+00:00",
|
||||||
|
"summary": {
|
||||||
|
"total_runs": 2,
|
||||||
|
"completed_runs": 2,
|
||||||
|
"overall_pass_rate": 0.75,
|
||||||
|
"overall_availability": 1.0,
|
||||||
|
"avg_latency_ms": 150.0,
|
||||||
|
},
|
||||||
|
"time_trend": [
|
||||||
|
{
|
||||||
|
"bucket_index": 0,
|
||||||
|
"start_seconds": 0.0,
|
||||||
|
"end_seconds": 3600.0,
|
||||||
|
"run_count": 2,
|
||||||
|
"pass_rate": 0.75,
|
||||||
|
"availability": 1.0,
|
||||||
|
"avg_latency_ms": 150.0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"bucket_index": 1,
|
||||||
|
"start_seconds": 3600.0,
|
||||||
|
"end_seconds": 7200.0,
|
||||||
|
"run_count": 0,
|
||||||
|
"pass_rate": None,
|
||||||
|
"availability": None,
|
||||||
|
"avg_latency_ms": None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"capability_summary": [
|
||||||
|
{
|
||||||
|
"scenario_id": "s-1",
|
||||||
|
"scenario_name": "售后场景",
|
||||||
|
"run_count": 2,
|
||||||
|
"pass_rate": 0.75,
|
||||||
|
"availability": 1.0,
|
||||||
|
"avg_latency_ms": 150.0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── render_html ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_render_html_contains_names_and_summary():
|
||||||
|
html = render_html(_run_report())
|
||||||
|
assert "客服机器人" in html
|
||||||
|
assert "售后场景" in html
|
||||||
|
assert "50.00%" in html # pass_rate 0.5
|
||||||
|
|
||||||
|
|
||||||
|
def test_render_html_contains_turns_and_rule_badges():
|
||||||
|
html = render_html(_run_report())
|
||||||
|
assert "退货流程" in html
|
||||||
|
assert "keyword_match" in html
|
||||||
|
assert "失败" in html
|
||||||
|
|
||||||
|
|
||||||
|
# ── render_markdown ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_render_markdown_summary_table():
|
||||||
|
md = render_markdown(_run_report())
|
||||||
|
assert "| 总用例数 | 2 |" in md
|
||||||
|
assert "| 通过率 | 50.0% |" in md
|
||||||
|
assert "| 连通用例 | 1 |" in md
|
||||||
|
|
||||||
|
|
||||||
|
def test_render_markdown_connectivity_badge_and_no_reply():
|
||||||
|
md = render_markdown(_run_report())
|
||||||
|
assert "🔗" in md # connectivity case badge
|
||||||
|
assert "（连通用例，未配置判定标准）" in md
|
||||||
|
assert "（无回复）" in md
|
||||||
|
|
||||||
|
|
||||||
|
def test_render_markdown_judged_pass_rate_dash_when_none():
|
||||||
|
report = _run_report()
|
||||||
|
report["summary"]["judged_pass_rate"] = None
|
||||||
|
md = render_markdown(report)
|
||||||
|
assert "| 判定型通过率 | — |" in md
|
||||||
|
|
||||||
|
|
||||||
|
# ── render_json ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_render_json_roundtrips():
|
||||||
|
report = _run_report()
|
||||||
|
parsed = json.loads(render_json(report))
|
||||||
|
assert parsed == report
|
||||||
|
|
||||||
|
|
||||||
|
# ── render_campaign_markdown ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_render_campaign_markdown_summary_and_axes():
|
||||||
|
md = render_campaign_markdown(_campaign_report())
|
||||||
|
assert "# 活动周期报告 — 夜间巡检" in md
|
||||||
|
assert "| 整窗通过率 | 75.0% |" in md
|
||||||
|
assert "## 时间趋势" in md
|
||||||
|
assert "## 能力汇总" in md
|
||||||
|
assert "| 售后场景 | 2 | 75.0% | 100.0% | 150ms |" in md
|
||||||
|
|
||||||
|
|
||||||
|
def test_render_campaign_markdown_empty_bucket_dashes():
|
||||||
|
md = render_campaign_markdown(_campaign_report())
|
||||||
|
# bucket 1 has no runs: pass_rate/availability/latency all render as —
|
||||||
|
assert "| 3600–7200 | 0 | — | — | — |" in md
|
||||||
Loading…
Reference in New Issue
Block a user