diff --git a/backend/agenteval/evaluation/report_render.py b/backend/agenteval/evaluation/report_render.py index 12ebac9..88a5762 100644 --- a/backend/agenteval/evaluation/report_render.py +++ b/backend/agenteval/evaluation/report_render.py @@ -7,6 +7,7 @@ show) belong to generation; this module only formats. """ import json +from datetime import datetime from typing import Any, Optional from jinja2 import Template @@ -213,25 +214,164 @@ def _render_analysis_lines(analysis: dict[str, Any], scenario_names: dict[str, s return lines +_CAMPAIGN_STATUS_LABELS = { + "planned": "计划中", "running": "进行中", "completed": "已完成", + "cancelled": "已取消", "failed": "失败", +} +_TREND_LABELS = {"improving": "改善", "stable": "平稳", "regressing": "退化"} +_EVOLUTION_LABELS = {"new": "新增", "persisting": "持续", "resolved": "消解"} +_TRACKING_LABELS = {"addressed": "已落实", "partial": "部分落实", "unaddressed": "未落实", "new": "新增"} + + +def _offset(seconds: float) -> str: + """Human-readable window offset: 86400 → 24h, 1800 → 30m, 45 → 45s.""" + s = int(round(seconds)) + if s % 3600 == 0: + return f"{s // 3600}h" + if s % 60 == 0: + return f"{s // 60}m" + return f"{s}s" + + +def _dt(iso: Optional[str]) -> str: + """ISO timestamp → `2026-08-01 08:00`; unparseable input passes through.""" + if not iso: + return "-" + try: + return datetime.fromisoformat(iso.replace("Z", "+00:00")).strftime("%Y-%m-%d %H:%M") + except ValueError: + return iso + + +def _diff_cell(metric: str, pair: dict[str, Any]) -> str: + """One metric cell of the diff table: `基线 → 本期(±delta)`.""" + + def _val(v: Optional[float]) -> str: + if v is None: + return "—" + return _ms(v) if metric == "avg_latency_ms" else _pct(v) + + delta = pair.get("delta") + if delta is None: + delta_text = "—" + elif metric == "avg_latency_ms": + delta_text = f"{delta:+.1f}ms" + else: + delta_text = f"{delta * 100:+.1f}pp" + return f"{_val(pair.get('baseline'))} → {_val(pair.get('current'))}({delta_text})" + + +def _render_comparison_lines(comparison: dict[str, Any], scenario_names: dict[str, str]) -> list[str]: + """Appendix for the campaign export: narrative + mechanical metric diff. + + ``comparison`` carries the completed narrative result plus context resolved + on the read path (baseline name/times, model name, generated-at, diff). + """ + + def _name(sid: str) -> str: + return scenario_names.get(sid, sid[:8]) + + result = comparison.get("result") or {} + lines = ["", "## 周期对比", ""] + meta: list[str] = [] + if comparison.get("baseline_name"): + base = f"基线:「{comparison['baseline_name']}」" + if comparison.get("baseline_completed_at"): + base += f"(完成于 {_dt(comparison['baseline_completed_at'])})" + meta.append(base) + if comparison.get("model_name"): + meta.append(f"分析模型:{comparison['model_name']}") + if comparison.get("updated_at"): + meta.append(f"生成于:{_dt(comparison['updated_at'])}") + if meta: + lines += [" · ".join(meta), ""] + + trend = _TREND_LABELS.get(result.get("trend"), "平稳") + lines += [f"**趋势**:{trend} — {result.get('summary') or '—'}", ""] + + diff = comparison.get("metric_diff") + if diff: + lines += [ + "### 指标变化", + "", + "| 维度 | 通过率 | 可用性 | 平均时延 |", + "|------|------|------|------|", + ] + + def _row(label: str, block: dict[str, Any]) -> str: + return ( + f"| {label} | {_diff_cell('pass_rate', block['pass_rate'])} | " + f"{_diff_cell('availability', block['availability'])} | " + f"{_diff_cell('avg_latency_ms', block['avg_latency_ms'])} |" + ) + + lines.append(_row("整窗(总体)", diff["overall"])) + for s in diff.get("scenarios") or []: + label = s.get("scenario_name") or _name(s.get("scenario_id", "")) + lines.append(_row(label, s)) + lines.append("") + + evolution = result.get("problem_evolution") or [] + if evolution: + lines += ["### 问题演变", ""] + for p in evolution: + status = _EVOLUTION_LABELS.get(p.get("status"), "持续") + names = "、".join(_name(sid) for sid in p.get("scenario_ids") or []) + head = f"- **[{status}] {p.get('title', '')}**" + if names: + head += f"(场景:{names})" + lines.append(head) + if p.get("detail"): + lines.append(f" {p['detail']}") + lines.append("") + + tracking = result.get("suggestion_tracking") or [] + if tracking: + lines += ["### 建议落实情况", ""] + for t in tracking: + status = _TRACKING_LABELS.get(t.get("status"), "未落实") + lines.append(f"- **[{status}] {t.get('text', '')}**") + if t.get("note"): + lines.append(f" {t['note']}") + lines.append("") + return lines + + +def _window_line(report: dict[str, Any]) -> str: + """Human-readable window line with the 正式线 / 加速调试线 wording.""" + window = report.get("window_seconds") or 0 + scale = float(report.get("time_scale") or 1) + if scale == 1: + return f"**窗口**: {_offset(window)}(正式线)" + wall = f",压缩后实际耗时约 {_offset(window / scale)}" + return f"**窗口**: {_offset(window)}(加速调试线 ×{scale:g}{wall})" + + def render_campaign_markdown( report: dict[str, Any], *, analysis: Optional[dict[str, Any]] = None, + comparison: Optional[dict[str, Any]] = None, + target_name: Optional[str] = None, scenario_names: Optional[dict[str, str]] = None, ) -> str: """Render a dual-axis campaign report dict as Markdown. - ``analysis`` is the stored 智能分析 result dict (completed only); when - absent the export is identical to the pre-analysis format. + ``analysis`` and ``comparison`` are the stored 智能分析 / 周期对比 results + (completed only); when absent the corresponding appendix is omitted + entirely. """ s = report["summary"] + status = _CAMPAIGN_STATUS_LABELS.get(report.get("status"), report.get("status") or "-") + target = target_name or report.get("target_id") or "-" lines: list[str] = [ f"# 活动周期报告 — {report['name']}", "", - f"**状态**: {report['status']} ", - f"**窗口**: {report['window_seconds']}s(倍速 {report['time_scale']}) ", - f"**开始时间**: {report['started_at'] or '-'} ", - f"**完成时间**: {report['completed_at'] or '-'} ", + f"**评测对象**: {target} ", + f"**状态**: {status} ", + _window_line(report) + " ", + f"**开始时间**: {_dt(report.get('started_at'))} ", + f"**完成时间**: {_dt(report.get('completed_at'))} ", "", "## 汇总", "", @@ -245,12 +385,12 @@ def render_campaign_markdown( "", "## 时间趋势", "", - "| 时段(秒) | 运行数 | 通过率 | 可用性 | 时延 |", + "| 时段 | 运行数 | 通过率 | 可用性 | 时延 |", "|------|------|------|------|------|", ] for b in report["time_trend"]: lines.append( - f"| {b['start_seconds']:.0f}–{b['end_seconds']:.0f} | {b['run_count']} | " + f"| {_offset(b['start_seconds'])}–{_offset(b['end_seconds'])} | {b['run_count']} | " f"{_pct(b['pass_rate'])} | {_pct(b['availability'])} | {_ms(b['avg_latency_ms'])} |" ) lines += [ @@ -267,4 +407,6 @@ def render_campaign_markdown( ) if analysis: lines += _render_analysis_lines(analysis, scenario_names or {}) + if comparison: + lines += _render_comparison_lines(comparison, scenario_names or {}) return "\n".join(lines) diff --git a/backend/agenteval/web/routers/campaigns.py b/backend/agenteval/web/routers/campaigns.py index 4eb2fb9..831b73e 100644 --- a/backend/agenteval/web/routers/campaigns.py +++ b/backend/agenteval/web/routers/campaigns.py @@ -134,9 +134,39 @@ async def get_campaign_report_markdown(campaign_id: str, session: Session = Depe scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()} analysis_row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id) analysis = analysis_row.get_result() if analysis_row and analysis_row.status == "completed" else None + target = TargetRepository(session).get(campaign.target_id) + target_name = target.name if target else None + + comparison = None + cmp_row = CampaignPeriodComparisonRepository(session).get_by_campaign(campaign_id) + if cmp_row is not None and cmp_row.status == "completed" and cmp_row.get_result(): + baseline = CampaignRepository(session).get(cmp_row.baseline_campaign_id) + metric_diff = ( + compute_metric_diff( + build_campaign_report_dict(baseline, session), + build_campaign_report_dict(campaign, session), + ) + if baseline is not None + else None + ) + model_cfg = ModelConfigRepository(session).get(cmp_row.model_config_id) if cmp_row.model_config_id else None + model_label = f"{model_cfg.name}({model_cfg.model_name})" if model_cfg and model_cfg.model_name else ( + model_cfg.name if model_cfg else None + ) + comparison = { + "result": cmp_row.get_result(), + "baseline_name": baseline.name if baseline else None, + "baseline_completed_at": iso_utc(baseline.completed_at) if baseline else None, + "model_name": model_label, + "updated_at": iso_utc(cmp_row.updated_at), + "metric_diff": metric_diff, + } + md = render_campaign_markdown( generate_campaign_report(campaign, runs, scenario_names=scenario_names), analysis=analysis, + comparison=comparison, + target_name=target_name, scenario_names=scenario_names, ) return Response( diff --git a/tests/integration/test_campaign_comparison_api.py b/tests/integration/test_campaign_comparison_api.py index 6d9ac64..5ddd8bd 100644 --- a/tests/integration/test_campaign_comparison_api.py +++ b/tests/integration/test_campaign_comparison_api.py @@ -407,3 +407,60 @@ async def test_post_comparison_rerun_upserts_without_new_row(client, seeded_db, select(CampaignPeriodComparisonDB).where(CampaignPeriodComparisonDB.campaign_id == current_id) ).all() assert len(rows) == 1 + + +# ── Markdown 导出纳入周期对比 ────────────────────────────────────────────── + + +async def test_markdown_export_includes_completed_comparison(client, seeded_db): + _seed_analysis_default(seeded_db) + baseline_id, current_id = await _two_period_setup(client, seeded_db) + _seed_run(seeded_db, "run-base", baseline_id, pass_rate=0.5, latency=800.0) + _seed_run(seeded_db, "run-cur", current_id, pass_rate=0.9, latency=500.0) + _complete_comparison_row(seeded_db, current_id, baseline_id) + + resp = await client.get(f"/api/campaigns/{current_id}/report/markdown") + assert resp.status_code == 200 + md = resp.text + + # 排版重优化:头部评测对象名、中文化状态、正式线窗口 + assert "**评测对象**: mock-target" in md + assert "**状态**: 已完成" in md + assert "**窗口**: 24h(正式线)" in md + + # 周期对比附录:元信息 + 趋势 + 机械 diff 表 + assert "## 周期对比" in md + assert "基线:「上期」" in md + assert "分析模型:analysis-cfg(m)" in md + assert "**趋势**:改善 — 通过率提升" in md + assert "| 整窗(总体) | 50.0% → 90.0%(+40.0pp) | 100.0% → 100.0%(+0.0pp) | 800ms → 500ms(-300.0ms) |" in md + assert "**[消解] 答非所问**(场景:mock-scenario)" in md + assert "**[已落实] 保持**" in md + + # 智能分析附录在前,周期对比紧随其后 + assert md.index("## 智能分析") < md.index("## 周期对比") + + +async def test_markdown_export_without_comparison_row_has_no_section(client, seeded_db): + _seed_analysis_default(seeded_db) + _, current_id = await _two_period_setup(client, seeded_db) + _seed_run(seeded_db, "run-cur", current_id, pass_rate=0.9, latency=500.0) + + resp = await client.get(f"/api/campaigns/{current_id}/report/markdown") + assert resp.status_code == 200 + assert "周期对比" not in resp.text + + +async def test_markdown_export_skips_non_completed_comparison(client, seeded_db): + _seed_analysis_default(seeded_db) + baseline_id, current_id = await _two_period_setup(client, seeded_db) + seeded_db.add(CampaignPeriodComparisonDB( + campaign_id=current_id, baseline_campaign_id=baseline_id, + status="failed", model_config_id="mc-1", triggered_by="manual", + error="boom", + )) + seeded_db.commit() + + resp = await client.get(f"/api/campaigns/{current_id}/report/markdown") + assert resp.status_code == 200 + assert "周期对比" not in resp.text diff --git a/tests/unit/test_report_render.py b/tests/unit/test_report_render.py index 19f245c..9619cc5 100644 --- a/tests/unit/test_report_render.py +++ b/tests/unit/test_report_render.py @@ -184,7 +184,7 @@ def test_render_campaign_markdown_summary_and_axes(): def test_render_campaign_markdown_empty_bucket_dashes(): md = render_campaign_markdown(_campaign_report()) # bucket 1 has no runs: pass_rate/availability/latency all render as — - assert "| 3600–7200 | 0 | — | — | — |" in md + assert "| 1h–2h | 0 | — | — | — |" in md def _analysis() -> dict: @@ -246,3 +246,103 @@ def test_render_campaign_markdown_analysis_falls_back_to_id_prefix(): def test_render_campaign_markdown_without_analysis_unchanged(): md = render_campaign_markdown(_campaign_report()) assert "智能分析" not in md + + +def test_render_campaign_markdown_header_readability(): + md = render_campaign_markdown(_campaign_report(), target_name="客服机器人") + assert "**评测对象**: 客服机器人" in md + assert "**状态**: 已完成" in md + assert "**窗口**: 2h(正式线)" in md + assert "**开始时间**: 2026-07-30 00:00" in md + assert "**完成时间**: 2026-07-30 02:00" in md + assert "| 0h–1h | 2 | 75.0% | 100.0% | 150ms |" in md + + +def test_render_campaign_markdown_header_falls_back_to_target_id(): + md = render_campaign_markdown(_campaign_report()) + assert "**评测对象**: t-1" in md + + +def test_render_campaign_markdown_accelerated_line(): + report = _campaign_report() + report["window_seconds"] = 86400 + report["time_scale"] = 4.0 + md = render_campaign_markdown(report) + assert "**窗口**: 24h(加速调试线 ×4,压缩后实际耗时约 6h)" in md + + +def _comparison() -> dict: + """A hand-built dict matching the comparison context the export endpoint resolves.""" + return { + "result": { + "trend": "regressing", + "summary": "整体质量下滑,售后场景恶化", + "problem_evolution": [ + { + "status": "persisting", + "title": "售后答非所问", + "scenario_ids": ["s-1"], + "detail": "问题仍未收敛", + }, + {"status": "resolved", "title": "响应偏慢", "scenario_ids": [], "detail": ""}, + ], + "suggestion_tracking": [ + {"status": "partial", "text": "补充售后知识库", "note": "仅覆盖部分问题"}, + {"status": "new", "text": "新增建议:监控时延", "note": ""}, + ], + }, + "baseline_name": "上一期巡检", + "baseline_completed_at": "2026-07-29T02:00:00+00:00", + "model_name": "qwen-max", + "updated_at": "2026-07-30T03:00:00+00:00", + "metric_diff": { + "overall": { + "pass_rate": {"baseline": 0.8, "current": 0.75, "delta": -0.05}, + "availability": {"baseline": 1.0, "current": 0.5, "delta": -0.5}, + "avg_latency_ms": {"baseline": 120.0, "current": 150.0, "delta": 30.0}, + }, + "scenarios": [ + { + "scenario_id": "s-1", + "scenario_name": "售后场景", + "pass_rate": {"baseline": 0.7, "current": 0.6, "delta": -0.1}, + "availability": {"baseline": None, "current": None, "delta": None}, + "avg_latency_ms": {"baseline": 100.0, "current": 140.0, "delta": 40.0}, + }, + ], + }, + } + + +def test_render_campaign_markdown_appends_comparison_section(): + md = render_campaign_markdown( + _campaign_report(), + comparison=_comparison(), + scenario_names={"s-1": "售后场景"}, + ) + assert "## 周期对比" in md + assert "基线:「上一期巡检」(完成于 2026-07-29 02:00)" in md + assert "分析模型:qwen-max" in md + assert "生成于:2026-07-30 03:00" in md + assert "**趋势**:退化 — 整体质量下滑,售后场景恶化" in md + assert "### 指标变化" in md + assert "| 整窗(总体) | 80.0% → 75.0%(-5.0pp) | 100.0% → 50.0%(-50.0pp) | 120ms → 150ms(+30.0ms) |" in md + assert "| 售后场景 | 70.0% → 60.0%(-10.0pp) | — → —(—) | 100ms → 140ms(+40.0ms) |" in md + assert "### 问题演变" in md + assert "**[持续] 售后答非所问**(场景:售后场景)" in md + assert " 问题仍未收敛" in md + assert "**[消解] 响应偏慢**" in md + assert "### 建议落实情况" in md + assert "**[部分落实] 补充售后知识库**" in md + assert " 仅覆盖部分问题" in md + assert "**[新增] 新增建议:监控时延**" in md + + +def test_render_campaign_markdown_comparison_comes_after_analysis(): + md = render_campaign_markdown(_campaign_report(), analysis=_analysis(), comparison=_comparison()) + assert md.index("## 智能分析") < md.index("## 周期对比") + + +def test_render_campaign_markdown_without_comparison_unchanged(): + md = render_campaign_markdown(_campaign_report()) + assert "周期对比" not in md