diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index a71e6d0..c6a7f96 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -1,95 +1,21 @@ -"""Report generation for evaluation runs.""" +"""Report generation for evaluation runs. + +Generation only: read the DB / model objects and build the report dict. +Formatting lives in ``report_render`` (pure dict → HTML/Markdown/JSON). +""" -import json from collections import defaultdict from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional -from jinja2 import Template - from agenteval.evaluation.metrics import aggregate_runs +from agenteval.evaluation.report_render import render_html, render_json, render_markdown from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary from agenteval.storage.db import DATA_DIR, iso_utc from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository from agenteval.utils.llm import extract_reply_text -HTML_TEMPLATE = """ - - - - 评测报告 - {{ report.run_id }} - - - -
-

评测报告

-

评测对象：{{ report.target_name }}（{{ report.target_id }}）

-

评测场景：{{ report.scenario_name }}（{{ report.scenario_id }}）

-

执行时间：{{ report.started_at }} 至 {{ report.completed_at or '进行中' }}

- -
-
-
{{ report.summary.total_cases }}
-
用例总数
-
-
-
{{ report.summary.passed_cases }}
-
通过用例
-
-
-
{{ report.summary.total_rules }}
-
规则总数
-
-
-
{{ "%.2f"|format(report.summary.pass_rate * 100) }}%
-
规则通过率
-
-
- - {% for case in report.cases %} -
-
用例 {{ case.case_id }}
- {% for turn in case.turns %} -
-
用户消息
- 
{{ turn.sent_text }}
-
智能体回复（{{ turn.latency_ms }}ms）
- 
{{ turn.reply_text or '（无回复）' }}
-
- {% endfor %} -
- {% for result in case.results %} -
- {{ '通过' if result.passed else '失败' }} - {{ result.rule_type }}: {{ result.reason }} -
- {% endfor %} -
-
- {% endfor %} -
- - -""" - def _extract_text(data: Any) -> str: return extract_reply_text(data) @@ -292,24 +218,19 @@ def _to_utc(dt: Optional[datetime]) -> Optional[datetime]: return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt -def _aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]: - """Delegates to the single cross-run aggregation seam (ADR-0004).""" - return aggregate_runs(runs) - - def summarize_campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict[str, Any]: """Compact list-row progress: completed vs *planned* total, plus pass_rate. Unlike ``campaign_progress`` (live window position), this powers the list view. ``planned_total`` is the sum of plan-entry counts — a fixed target the campaign works toward, so the progress bar fills from 0 rather than tracking - a growing spawned count. ``overall_pass_rate`` reuses ``_aggregate_runs`` so + a growing spawned count. ``overall_pass_rate`` reuses ``aggregate_runs`` so the list figure matches the report exactly (ADR-0002: failures count as 0.0). """ return { "completed_runs": sum(1 for r in runs if r.status == RunStatus.COMPLETED), "planned_total": sum(entry.count for entry in campaign.plan), - "overall_pass_rate": _aggregate_runs(runs)["pass_rate"], + "overall_pass_rate": aggregate_runs(runs)["pass_rate"], } @@ -350,7 +271,7 @@ def generate_campaign_report( time_trend = [] for idx in range(bucket_count): - agg = _aggregate_runs(buckets.get(idx, [])) + agg = aggregate_runs(buckets.get(idx, [])) time_trend.append({ "bucket_index": idx, "start_seconds": round(idx * bucket_seconds, 3), @@ -364,7 +285,7 @@ def generate_campaign_report( by_scenario[run.scenario_id].append(run) capability_summary = [] for sid, sruns in by_scenario.items(): - agg = _aggregate_runs(sruns) + agg = aggregate_runs(sruns) capability_summary.append({ "scenario_id": sid, "scenario_name": scenario_names.get(sid, (sid or "")[:8]), @@ -372,7 +293,7 @@ def generate_campaign_report( }) capability_summary.sort(key=lambda s: s["run_count"], reverse=True) - overall = _aggregate_runs(runs) + overall = aggregate_runs(runs) return { "campaign_id": campaign.id, "name": campaign.name, @@ -394,163 +315,22 @@ def generate_campaign_report( } -def render_campaign_markdown_report( - campaign: Campaign, - runs: list[EvalRun], - *, - scenario_names: Optional[dict[str, str]] = None, -) -> str: - """Render the dual-axis campaign report as Markdown (reuses the export path).""" - report = generate_campaign_report(campaign, runs, scenario_names=scenario_names) - s = report["summary"] - - def _pct(v: Optional[float]) -> str: - return "—" if v is None else f"{v * 100:.1f}%" - - def _ms(v: Optional[float]) -> str: - return "—" if v is None else f"{v:.0f}ms" - - lines: list[str] = [ - f"# 活动周期报告 — {report['name']}", - "", - f"**状态**: {report['status']} ", - f"**窗口**: {report['window_seconds']}s（倍速 {report['time_scale']}） ", - f"**开始时间**: {report['started_at'] or '-'} ", - f"**完成时间**: {report['completed_at'] or '-'} ", - "", - "## 汇总", - "", - "| 指标 | 数值 |", - "|------|------|", - f"| 子运行总数 | {s['total_runs']} |", - f"| 已完成 | {s['completed_runs']} |", - f"| 整窗通过率 | {_pct(s['overall_pass_rate'])} |", - f"| 整窗可用性 | {_pct(s['overall_availability'])} |", - f"| 平均时延 | {_ms(s['avg_latency_ms'])} |", - "", - "## 时间趋势", - "", - "| 时段(秒) | 运行数 | 通过率 | 可用性 | 时延 |", - "|------|------|------|------|------|", - ] - for b in report["time_trend"]: - lines.append( - f"| {b['start_seconds']:.0f}–{b['end_seconds']:.0f} | {b['run_count']} | " - f"{_pct(b['pass_rate'])} | {_pct(b['availability'])} | {_ms(b['avg_latency_ms'])} |" - ) - lines += [ - "", - "## 能力汇总", - "", - "| 场景 | 运行数 | 通过率 | 可用性 | 时延 |", - "|------|------|------|------|------|", - ] - for c in report["capability_summary"]: - lines.append( - f"| {c['scenario_name']} | {c['run_count']} | {_pct(c['pass_rate'])} | " - f"{_pct(c['availability'])} | {_ms(c['avg_latency_ms'])} |" - ) - return "\n".join(lines) - - -def render_markdown_report(run_id: str, session=None) -> str: - """Render a report as Markdown string.""" - report = generate_report(run_id, session) - s = report["summary"] - judged_rate = s.get("judged_pass_rate") - judged_rate_text = "—" if judged_rate is None else f"{judged_rate * 100:.1f}%" - lines: list[str] = [ - f"# 评测报告 — {report.get('scenario_name', run_id)}", - "", - f"**评测对象**: {report.get('target_name', '-')} ", - f"**评测场景**: {report.get('scenario_name', '-')} ", - f"**状态**: {report.get('status', '-')} ", - f"**开始时间**: {report.get('started_at', '-')} ", - f"**完成时间**: {report.get('completed_at', '-')} ", - "", - "## 汇总", - "", - "| 指标 | 数值 |", - "|------|------|", - f"| 总用例数 | {s['total_cases']} |", - f"| 通过用例 | {s['passed_cases']} |", - f"| 失败用例 | {s['failed_cases']} |", - f"| 总规则数 | {s['total_rules']} |", - f"| 通过规则 | {s['passed_rules']} |", - f"| 通过率 | {s['pass_rate'] * 100:.1f}% |", - f"| 连通用例 | {s.get('connectivity_cases', 0)} |", - f"| 判定型通过率 | {judged_rate_text} |", - "", - "## 用例明细", - "", - ] - - for case in report.get("cases", []): - if case.get("connectivity"): - badge = "🔗" - else: - badge = "✅" if case.get("passed") else "❌" - title = f"### {badge} 用例 `{case['case_id']}`" - if case.get("connectivity"): - title += "（连通用例，未配置判定标准）" - lines.append(title) - lines.append("") - - for turn in case.get("turns", []): - lines.append(f"**第 {turn['round']} 轮**") - lines.append("") - lines.append(f"> **用户**: {turn.get('sent_text', '—')}") - lines.append("") - reply = turn.get("reply_text") or "（无回复）" - lines.append(f"> **智能体**: {reply}") - if turn.get("latency_ms") is not None: - lines.append(f"> *延迟: {turn['latency_ms']}ms*") - lines.append("") - - if case.get("results"): - lines.append("**规则评估结果**") - lines.append("") - lines.append("| 规则 | 结果 | 评分 | 说明 |") - lines.append("|------|------|------|------|") - for r in case["results"]: - badge = "✅" if r["passed"] else "❌" - score = f"{r['score']:.2f}" if r.get("score") is not None else "-" - lines.append(f"| {r['rule_type']} | {badge} | {score} | {r.get('reason', '')} |") - lines.append("") - - return "\n".join(lines) - - -def render_json_report(run_id: str, session=None) -> str: - """Render a report as JSON string.""" - report = generate_report(run_id, session) - return json.dumps(report, ensure_ascii=False, indent=2) - - -def render_html_report(run_id: str, session=None) -> str: - """Render a report as HTML string.""" - report = generate_report(run_id, session) - template = Template(HTML_TEMPLATE) - return template.render(report=report) - - def save_report(run_id: str, fmt: str = "html", output_dir: Optional[Path] = None) -> Path: - """Generate and save a report to disk.""" + """Generate a run report and save it to disk in the requested format.""" output_dir = output_dir or DATA_DIR / "reports" output_dir.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - if fmt == "html": - content = render_html_report(run_id) - path = output_dir / f"report_{run_id}_{timestamp}.html" - elif fmt == "json": - content = render_json_report(run_id) - path = output_dir / f"report_{run_id}_{timestamp}.json" - elif fmt == "markdown": - content = render_markdown_report(run_id) - path = output_dir / f"report_{run_id}_{timestamp}.md" - else: + report = generate_report(run_id) + renderers = { + "html": (render_html, "html"), + "json": (render_json, "json"), + "markdown": (render_markdown, "md"), + } + if fmt not in renderers: raise ValueError(f"unsupported report format: {fmt}") + render, ext = renderers[fmt] - path.write_text(content, encoding="utf-8") + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + path = output_dir / f"report_{run_id}_{timestamp}.{ext}" + path.write_text(render(report), encoding="utf-8") return path diff --git a/backend/agenteval/evaluation/report_render.py b/backend/agenteval/evaluation/report_render.py new file mode 100644 index 0000000..1dbd6f4 --- /dev/null +++ b/backend/agenteval/evaluation/report_render.py @@ -0,0 +1,219 @@ +"""Pure renderers: report dict in, HTML/Markdown/JSON string out. + +No I/O and no storage access — every function takes the dict produced by the +generation side (``evaluation.report``) so rendering can be unit-tested from a +hand-built dict. Report *content* decisions (which cases pass, which rates to +show) belong to generation; this module only formats. +""" + +import json +from typing import Any, Optional + +from jinja2 import Template + +HTML_TEMPLATE = """ + + + + 评测报告 - {{ report.run_id }} + + + +
+

评测报告

+

评测对象：{{ report.target_name }}（{{ report.target_id }}）

+

评测场景：{{ report.scenario_name }}（{{ report.scenario_id }}）

+

执行时间：{{ report.started_at }} 至 {{ report.completed_at or '进行中' }}

+ +
+
+
{{ report.summary.total_cases }}
+
用例总数
+
+
+
{{ report.summary.passed_cases }}
+
通过用例
+
+
+
{{ report.summary.total_rules }}
+
规则总数
+
+
+
{{ "%.2f"|format(report.summary.pass_rate * 100) }}%
+
规则通过率
+
+
+ + {% for case in report.cases %} +
+
用例 {{ case.case_id }}
+ {% for turn in case.turns %} +
+
用户消息
+ 
{{ turn.sent_text }}
+
智能体回复（{{ turn.latency_ms }}ms）
+ 
{{ turn.reply_text or '（无回复）' }}
+
+ {% endfor %} +
+ {% for result in case.results %} +
+ {{ '通过' if result.passed else '失败' }} + {{ result.rule_type }}: {{ result.reason }} +
+ {% endfor %} +
+
+ {% endfor %} +
+ + +""" + + +def _pct(v: Optional[float]) -> str: + return "—" if v is None else f"{v * 100:.1f}%" + + +def _ms(v: Optional[float]) -> str: + return "—" if v is None else f"{v:.0f}ms" + + +def render_html(report: dict[str, Any]) -> str: + """Render a run report dict as an HTML string.""" + return Template(HTML_TEMPLATE).render(report=report) + + +def render_json(report: dict[str, Any]) -> str: + """Render a report dict as a JSON string.""" + return json.dumps(report, ensure_ascii=False, indent=2) + + +def render_markdown(report: dict[str, Any]) -> str: + """Render a run report dict as Markdown.""" + s = report["summary"] + judged_rate = s.get("judged_pass_rate") + judged_rate_text = "—" if judged_rate is None else f"{judged_rate * 100:.1f}%" + lines: list[str] = [ + f"# 评测报告 — {report.get('scenario_name', report.get('run_id', ''))}", + "", + f"**评测对象**: {report.get('target_name', '-')} ", + f"**评测场景**: {report.get('scenario_name', '-')} ", + f"**状态**: {report.get('status', '-')} ", + f"**开始时间**: {report.get('started_at', '-')} ", + f"**完成时间**: {report.get('completed_at', '-')} ", + "", + "## 汇总", + "", + "| 指标 | 数值 |", + "|------|------|", + f"| 总用例数 | {s['total_cases']} |", + f"| 通过用例 | {s['passed_cases']} |", + f"| 失败用例 | {s['failed_cases']} |", + f"| 总规则数 | {s['total_rules']} |", + f"| 通过规则 | {s['passed_rules']} |", + f"| 通过率 | {s['pass_rate'] * 100:.1f}% |", + f"| 连通用例 | {s.get('connectivity_cases', 0)} |", + f"| 判定型通过率 | {judged_rate_text} |", + "", + "## 用例明细", + "", + ] + + for case in report.get("cases", []): + if case.get("connectivity"): + badge = "🔗" + else: + badge = "✅" if case.get("passed") else "❌" + title = f"### {badge} 用例 `{case['case_id']}`" + if case.get("connectivity"): + title += "（连通用例，未配置判定标准）" + lines.append(title) + lines.append("") + + for turn in case.get("turns", []): + lines.append(f"**第 {turn['round']} 轮**") + lines.append("") + lines.append(f"> **用户**: {turn.get('sent_text', '—')}") + lines.append("") + reply = turn.get("reply_text") or "（无回复）" + lines.append(f"> **智能体**: {reply}") + if turn.get("latency_ms") is not None: + lines.append(f"> *延迟: {turn['latency_ms']}ms*") + lines.append("") + + if case.get("results"): + lines.append("**规则评估结果**") + lines.append("") + lines.append("| 规则 | 结果 | 评分 | 说明 |") + lines.append("|------|------|------|------|") + for r in case["results"]: + badge = "✅" if r["passed"] else "❌" + score = f"{r['score']:.2f}" if r.get("score") is not None else "-" + lines.append(f"| {r['rule_type']} | {badge} | {score} | {r.get('reason', '')} |") + lines.append("") + + return "\n".join(lines) + + +def render_campaign_markdown(report: dict[str, Any]) -> str: + """Render a dual-axis campaign report dict as Markdown.""" + s = report["summary"] + lines: list[str] = [ + f"# 活动周期报告 — {report['name']}", + "", + f"**状态**: {report['status']} ", + f"**窗口**: {report['window_seconds']}s（倍速 {report['time_scale']}） ", + f"**开始时间**: {report['started_at'] or '-'} ", + f"**完成时间**: {report['completed_at'] or '-'} ", + "", + "## 汇总", + "", + "| 指标 | 数值 |", + "|------|------|", + f"| 子运行总数 | {s['total_runs']} |", + f"| 已完成 | {s['completed_runs']} |", + f"| 整窗通过率 | {_pct(s['overall_pass_rate'])} |", + f"| 整窗可用性 | {_pct(s['overall_availability'])} |", + f"| 平均时延 | {_ms(s['avg_latency_ms'])} |", + "", + "## 时间趋势", + "", + "| 时段(秒) | 运行数 | 通过率 | 可用性 | 时延 |", + "|------|------|------|------|------|", + ] + for b in report["time_trend"]: + lines.append( + f"| {b['start_seconds']:.0f}–{b['end_seconds']:.0f} | {b['run_count']} | " + f"{_pct(b['pass_rate'])} | {_pct(b['availability'])} | {_ms(b['avg_latency_ms'])} |" + ) + lines += [ + "", + "## 能力汇总", + "", + "| 场景 | 运行数 | 通过率 | 可用性 | 时延 |", + "|------|------|------|------|------|", + ] + for c in report["capability_summary"]: + lines.append( + f"| {c['scenario_name']} | {c['run_count']} | {_pct(c['pass_rate'])} | " + f"{_pct(c['availability'])} | {_ms(c['avg_latency_ms'])} |" + ) + return "\n".join(lines) diff --git a/backend/agenteval/web/routers/campaigns.py b/backend/agenteval/web/routers/campaigns.py index ac18a51..055c3a9 100644 --- a/backend/agenteval/web/routers/campaigns.py +++ b/backend/agenteval/web/routers/campaigns.py @@ -12,11 +12,8 @@ from pydantic import BaseModel, Field from sqlmodel import Session from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign -from agenteval.evaluation.report import ( - generate_campaign_report, - render_campaign_markdown_report, - summarize_campaign_progress, -) +from agenteval.evaluation.report import generate_campaign_report, summarize_campaign_progress +from agenteval.evaluation.report_render import render_campaign_markdown from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus from agenteval.storage.db import utc_now from agenteval.storage.repository import ( @@ -114,7 +111,7 @@ async def get_campaign_report_markdown(campaign_id: str, session: Session = Depe raise HTTPException(status_code=404, detail="campaign not found") runs = RunRepository(session).list_by_campaign(campaign_id) scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()} - md = render_campaign_markdown_report(campaign, runs, scenario_names=scenario_names) + md = render_campaign_markdown(generate_campaign_report(campaign, runs, scenario_names=scenario_names)) return Response( content=md, media_type="text/markdown; charset=utf-8", diff --git a/backend/agenteval/web/routers/reports.py b/backend/agenteval/web/routers/reports.py index 8e1ae16..ec0f55a 100644 --- a/backend/agenteval/web/routers/reports.py +++ b/backend/agenteval/web/routers/reports.py @@ -3,13 +3,8 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Response from sqlmodel import Session -from agenteval.evaluation.report import ( - generate_compare_report, - generate_report, - render_html_report, - render_json_report, - render_markdown_report, -) +from agenteval.evaluation.report import generate_compare_report, generate_report +from agenteval.evaluation.report_render import render_html, render_json, render_markdown from agenteval.storage.repository import RunRepository from agenteval.web.deps import get_db @@ -53,7 +48,7 @@ def get_html_report(run_id: str, session: Session = Depends(get_db)) -> Response run = RunRepository(session).get(run_id) if not run: raise HTTPException(status_code=404, detail="run not found") - html = render_html_report(run_id, session) + html = render_html(generate_report(run_id, session)) return Response(content=html, media_type="text/html") @@ -62,7 +57,7 @@ def get_json_report(run_id: str, session: Session = Depends(get_db)) -> Response run = RunRepository(session).get(run_id) if not run: raise HTTPException(status_code=404, detail="run not found") - json_text = render_json_report(run_id, session) + json_text = render_json(generate_report(run_id, session)) return Response(content=json_text, media_type="application/json") @@ -71,7 +66,7 @@ def get_markdown_report(run_id: str, session: Session = Depends(get_db)) -> Resp run = RunRepository(session).get(run_id) if not run: raise HTTPException(status_code=404, detail="run not found") - md = render_markdown_report(run_id, session) + md = render_markdown(generate_report(run_id, session)) return Response( content=md, media_type="text/markdown; charset=utf-8", diff --git a/backend/cli/report.py b/backend/cli/report.py index f4c2eb8..1f61a14 100644 --- a/backend/cli/report.py +++ b/backend/cli/report.py @@ -4,7 +4,9 @@ from pathlib import Path from typing import Optional import typer -from agenteval.evaluation.report import render_html_report, render_json_report, save_report +from agenteval.evaluation.report import generate_report as build_report +from agenteval.evaluation.report import save_report +from agenteval.evaluation.report_render import render_html, render_json from agenteval.storage.repository import RunRepository from rich.console import Console @@ -23,9 +25,9 @@ def show_report( raise typer.Exit(1) if fmt == "json": - console.print(render_json_report(run_id)) + console.print(render_json(build_report(run_id))) elif fmt == "html": - console.print(render_html_report(run_id)) + console.print(render_html(build_report(run_id))) else: console.print(f"不支持的格式: {fmt}", style="red") raise typer.Exit(1) @@ -51,16 +53,14 @@ def compare_reports( run_id_1: str, run_id_2: str, ) -> None: - from agenteval.evaluation.report import generate_report - r1 = RunRepository().get(run_id_1) r2 = RunRepository().get(run_id_2) if not r1 or not r2: console.print("评测记录不存在", style="red") raise typer.Exit(1) - report1 = generate_report(run_id_1) - report2 = generate_report(run_id_2) + report1 = build_report(run_id_1) + report2 = build_report(run_id_2) console.print(f"对比: {run_id_1} vs {run_id_2}") console.print(f" 用例数: {report1['summary']['total_cases']} -> {report2['summary']['total_cases']}") diff --git a/backend/cli/run.py b/backend/cli/run.py index be595f9..5f66353 100644 --- a/backend/cli/run.py +++ b/backend/cli/run.py @@ -57,9 +57,10 @@ def start_run( console.print_json(data=run.summary) if output_json: - from agenteval.evaluation.report import render_json_report + from agenteval.evaluation.report import generate_report + from agenteval.evaluation.report_render import render_json - console.print(render_json_report(run.id)) + console.print(render_json(generate_report(run.id))) @app.command("list", help="列出评测执行记录") diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py index a52ba86..d30fd78 100644 --- a/tests/unit/test_report.py +++ b/tests/unit/test_report.py @@ -6,9 +6,8 @@ from sqlmodel import Session, SQLModel, create_engine from agenteval.evaluation.report import ( generate_compare_report, generate_report, - render_markdown_report, - render_json_report, ) +from agenteval.evaluation.report_render import render_json, render_markdown from agenteval.models import ( Case, CaseType, EvalResult, EvalRun, EvalTarget, RunStatus, Scenario, Turn, PlatformType, ChannelType, TargetStatus, @@ -318,7 +317,7 @@ def test_compare_report_marks_connectivity(report_session): def test_markdown_report_shows_connectivity(report_session): run_id = _seed_run(report_session, n_cases=1, connectivity_cases=1) - md = render_markdown_report(run_id, report_session) + md = render_markdown(generate_report(run_id, report_session)) assert "连通用例" in md assert "判定型通过率" in md @@ -327,13 +326,13 @@ def test_markdown_report_shows_connectivity(report_session): def test_render_markdown_contains_header(report_session): run_id = _seed_run(report_session) - md = render_markdown_report(run_id, report_session) + md = render_markdown(generate_report(run_id, report_session)) assert "# 评测报告" in md def test_render_markdown_contains_summary_table(report_session): run_id = _seed_run(report_session) - md = render_markdown_report(run_id, report_session) + md = render_markdown(generate_report(run_id, report_session)) assert "## 汇总" in md assert "| 指标 | 数值 |" in md assert "通过率" in md @@ -341,14 +340,14 @@ def test_render_markdown_contains_summary_table(report_session): def test_render_markdown_contains_case_section(report_session): run_id = _seed_run(report_session, n_cases=1) - md = render_markdown_report(run_id, report_session) + md = render_markdown(generate_report(run_id, report_session)) assert "## 用例明细" in md assert "### " in md # case header def test_render_markdown_contains_rule_table(report_session): run_id = _seed_run(report_session) - md = render_markdown_report(run_id, report_session) + md = render_markdown(generate_report(run_id, report_session)) assert "**规则评估结果**" in md assert "keyword_match" in md @@ -358,7 +357,7 @@ def test_render_markdown_contains_rule_table(report_session): def test_render_json_report_is_valid_json(report_session): import json run_id = _seed_run(report_session) - json_text = render_json_report(run_id, report_session) + json_text = render_json(generate_report(run_id, report_session)) parsed = json.loads(json_text) assert parsed["run_id"] == run_id @@ -380,7 +379,7 @@ def test_case_dict_contains_passed_fallback(report_session): def test_markdown_errored_case_shows_failed_badge(report_session): """故障用例（无结果且非连通）在 MD 中必须 ❌ —— 此前 all([]) 误判 ✅。""" run_id = _seed_run(report_session, n_cases=1, errored_cases=1) - md = render_markdown_report(run_id, report_session) + md = render_markdown(generate_report(run_id, report_session)) assert "❌ 用例 `err0`" in md assert "✅ 用例 `err0`" not in md @@ -396,7 +395,7 @@ def test_authoritative_case_outcomes_override_reconstruction(report_session): report = generate_report(run_id, report_session) assert report["cases"][0]["passed"] is False - md = render_markdown_report(run_id, report_session) + md = render_markdown(generate_report(run_id, report_session)) assert "❌ 用例 `c0`" in md diff --git a/tests/unit/test_report_render.py b/tests/unit/test_report_render.py new file mode 100644 index 0000000..817f0f1 --- /dev/null +++ b/tests/unit/test_report_render.py @@ -0,0 +1,187 @@ +"""Unit tests for the pure renderers: dict in, HTML/Markdown/JSON out (no DB).""" + +import json + +from agenteval.evaluation.report_render import ( + render_campaign_markdown, + render_html, + render_json, + render_markdown, +) + + +def _run_report(**overrides) -> dict: + """A hand-built report dict matching generate_report's shape.""" + report = { + "run_id": "run-1", + "target_id": "t-1", + "target_name": "客服机器人", + "scenario_id": "s-1", + "scenario_name": "售后场景", + "scenario_version": 2, + "triggered_by": "manual", + "status": "completed", + "started_at": "2026-07-30T10:00:00+00:00", + "completed_at": "2026-07-30T10:05:00+00:00", + "summary": { + "total_cases": 2, + "passed_cases": 1, + "failed_cases": 1, + "total_rules": 3, + "passed_rules": 2, + "pass_rate": 0.5, + "connectivity_cases": 1, + "judged_pass_rate": 0.0, + }, + "cases": [ + { + "case_id": "case-a", + "passed": True, + "connectivity": True, + "turns": [ + { + "round": 0, + "sent_text": "你好", + "reply_text": "您好，请问有什么可以帮您？", + "latency_ms": 120, + "question_msg_id": "m-1", + } + ], + "results": [], + }, + { + "case_id": "case-b", + "passed": False, + "connectivity": False, + "turns": [ + { + "round": 0, + "sent_text": "退货流程", + "reply_text": None, + "latency_ms": None, + "question_msg_id": "m-2", + } + ], + "results": [ + {"rule_type": "keyword_match", "passed": False, "score": 0.0, "reason": "缺少关键词"}, + ], + }, + ], + } + report.update(overrides) + return report + + +def _campaign_report() -> dict: + """A hand-built dict matching generate_campaign_report's shape.""" + return { + "campaign_id": "c-1", + "name": "夜间巡检", + "target_id": "t-1", + "status": "completed", + "window_seconds": 7200, + "time_scale": 1.0, + "started_at": "2026-07-30T00:00:00+00:00", + "completed_at": "2026-07-30T02:00:00+00:00", + "summary": { + "total_runs": 2, + "completed_runs": 2, + "overall_pass_rate": 0.75, + "overall_availability": 1.0, + "avg_latency_ms": 150.0, + }, + "time_trend": [ + { + "bucket_index": 0, + "start_seconds": 0.0, + "end_seconds": 3600.0, + "run_count": 2, + "pass_rate": 0.75, + "availability": 1.0, + "avg_latency_ms": 150.0, + }, + { + "bucket_index": 1, + "start_seconds": 3600.0, + "end_seconds": 7200.0, + "run_count": 0, + "pass_rate": None, + "availability": None, + "avg_latency_ms": None, + }, + ], + "capability_summary": [ + { + "scenario_id": "s-1", + "scenario_name": "售后场景", + "run_count": 2, + "pass_rate": 0.75, + "availability": 1.0, + "avg_latency_ms": 150.0, + }, + ], + } + + +# ── render_html ───────────────────────────────────────────────────────────── + +def test_render_html_contains_names_and_summary(): + html = render_html(_run_report()) + assert "客服机器人" in html + assert "售后场景" in html + assert "50.00%" in html # pass_rate 0.5 + + +def test_render_html_contains_turns_and_rule_badges(): + html = render_html(_run_report()) + assert "退货流程" in html + assert "keyword_match" in html + assert "失败" in html + + +# ── render_markdown ───────────────────────────────────────────────────────── + +def test_render_markdown_summary_table(): + md = render_markdown(_run_report()) + assert "| 总用例数 | 2 |" in md + assert "| 通过率 | 50.0% |" in md + assert "| 连通用例 | 1 |" in md + + +def test_render_markdown_connectivity_badge_and_no_reply(): + md = render_markdown(_run_report()) + assert "🔗" in md # connectivity case badge + assert "（连通用例，未配置判定标准）" in md + assert "（无回复）" in md + + +def test_render_markdown_judged_pass_rate_dash_when_none(): + report = _run_report() + report["summary"]["judged_pass_rate"] = None + md = render_markdown(report) + assert "| 判定型通过率 | — |" in md + + +# ── render_json ───────────────────────────────────────────────────────────── + +def test_render_json_roundtrips(): + report = _run_report() + parsed = json.loads(render_json(report)) + assert parsed == report + + +# ── render_campaign_markdown ──────────────────────────────────────────────── + +def test_render_campaign_markdown_summary_and_axes(): + md = render_campaign_markdown(_campaign_report()) + assert "# 活动周期报告 — 夜间巡检" in md + assert "| 整窗通过率 | 75.0% |" in md + assert "## 时间趋势" in md + assert "## 能力汇总" in md + assert "| 售后场景 | 2 | 75.0% | 100.0% | 150ms |" in md + + +def test_render_campaign_markdown_empty_bucket_dashes(): + md = render_campaign_markdown(_campaign_report()) + # bucket 1 has no runs: pass_rate/availability/latency all render as — + assert "| 3600–7200 | 0 | — | — | — |" in md