"""Unit tests for the pure renderers: dict in, HTML/Markdown/JSON out (no DB).""" import json from agenteval.evaluation.report_render import ( render_campaign_markdown, render_html, render_json, render_markdown, ) def _run_report(**overrides) -> dict: """A hand-built report dict matching generate_report's shape.""" report = { "run_id": "run-1", "target_id": "t-1", "target_name": "客服机器人", "scenario_id": "s-1", "scenario_name": "售后场景", "scenario_version": 2, "triggered_by": "manual", "status": "completed", "started_at": "2026-07-30T10:00:00+00:00", "completed_at": "2026-07-30T10:05:00+00:00", "summary": { "total_cases": 2, "passed_cases": 1, "failed_cases": 1, "total_rules": 3, "passed_rules": 2, "pass_rate": 0.5, "connectivity_cases": 1, "judged_pass_rate": 0.0, }, "cases": [ { "case_id": "case-a", "passed": True, "connectivity": True, "turns": [ { "round": 0, "sent_text": "你好", "reply_text": "您好,请问有什么可以帮您?", "latency_ms": 120, "question_msg_id": "m-1", } ], "results": [], }, { "case_id": "case-b", "passed": False, "connectivity": False, "turns": [ { "round": 0, "sent_text": "退货流程", "reply_text": None, "latency_ms": None, "question_msg_id": "m-2", } ], "results": [ {"rule_type": "keyword_match", "passed": False, "score": 0.0, "reason": "缺少关键词"}, ], }, ], } report.update(overrides) return report def _campaign_report() -> dict: """A hand-built dict matching generate_campaign_report's shape.""" return { "campaign_id": "c-1", "name": "夜间巡检", "target_id": "t-1", "status": "completed", "window_seconds": 7200, "time_scale": 1.0, "started_at": "2026-07-30T00:00:00+00:00", "completed_at": "2026-07-30T02:00:00+00:00", "summary": { "total_runs": 2, "completed_runs": 2, "overall_pass_rate": 0.75, "overall_availability": 1.0, "avg_latency_ms": 150.0, }, "time_trend": [ { "bucket_index": 0, "start_seconds": 0.0, "end_seconds": 3600.0, "run_count": 2, "pass_rate": 0.75, "availability": 1.0, "avg_latency_ms": 150.0, }, { "bucket_index": 1, "start_seconds": 3600.0, "end_seconds": 7200.0, "run_count": 0, "pass_rate": None, "availability": None, "avg_latency_ms": None, }, ], "capability_summary": [ { "scenario_id": "s-1", "scenario_name": "售后场景", "run_count": 2, "pass_rate": 0.75, "availability": 1.0, "avg_latency_ms": 150.0, }, ], } # ── render_html ───────────────────────────────────────────────────────────── def test_render_html_contains_names_and_summary(): html = render_html(_run_report()) assert "客服机器人" in html assert "售后场景" in html assert "50.00%" in html # pass_rate 0.5 def test_render_html_contains_turns_and_rule_badges(): html = render_html(_run_report()) assert "退货流程" in html assert "keyword_match" in html assert "失败" in html # ── render_markdown ───────────────────────────────────────────────────────── def test_render_markdown_summary_table(): md = render_markdown(_run_report()) assert "| 总用例数 | 2 |" in md assert "| 通过率 | 50.0% |" in md assert "| 连通用例 | 1 |" in md def test_render_markdown_connectivity_badge_and_no_reply(): md = render_markdown(_run_report()) assert "🔗" in md # connectivity case badge assert "(连通用例,未配置判定标准)" in md assert "(无回复)" in md def test_render_markdown_judged_pass_rate_dash_when_none(): report = _run_report() report["summary"]["judged_pass_rate"] = None md = render_markdown(report) assert "| 判定型通过率 | — |" in md # ── render_json ───────────────────────────────────────────────────────────── def test_render_json_roundtrips(): report = _run_report() parsed = json.loads(render_json(report)) assert parsed == report # ── render_campaign_markdown ──────────────────────────────────────────────── def test_render_campaign_markdown_summary_and_axes(): md = render_campaign_markdown(_campaign_report()) assert "# 活动周期报告 — 夜间巡检" in md assert "| 整窗通过率 | 75.0% |" in md assert "## 时间趋势" in md assert "## 能力汇总" in md assert "| 售后场景 | 2 | 75.0% | 100.0% | 150ms |" in md def test_render_campaign_markdown_empty_bucket_dashes(): md = render_campaign_markdown(_campaign_report()) # bucket 1 has no runs: pass_rate/availability/latency all render as — assert "| 1h–2h | 0 | — | — | — |" in md def _analysis() -> dict: """A hand-built dict matching the stored campaign analysis result shape.""" return { "overall": "整窗通过率偏低,售后场景拖后腿", "problems": [ { "severity": "high", "title": "售后答非所问", "description": "多轮对话中反复偏离用户问题", "scenario_ids": ["s-1"], "evidence_run_ids": ["run-abc", "run-def"], }, { "severity": "low", "title": "响应偏慢", "description": "高峰时段时延偏高", "scenario_ids": ["s-2"], "evidence_run_ids": [], }, ], "scenario_narratives": [ {"scenario_id": "s-1", "narrative": "售后场景表现不稳定"}, {"scenario_id": "s-2", "narrative": "售前场景表现稳定"}, ], "suggestions": [ {"priority": 2, "text": "次要建议:扩容"}, {"priority": 1, "text": "首要建议:补充售后知识库"}, ], } def test_render_campaign_markdown_appends_analysis_sections(): md = render_campaign_markdown( _campaign_report(), analysis=_analysis(), scenario_names={"s-1": "售后场景", "s-2": "售前场景"}, ) assert "## 智能分析" in md assert "### 总体结论" in md assert "整窗通过率偏低,售后场景拖后腿" in md assert "### 问题诊断" in md assert "**[高] 售后答非所问**(场景:售后场景)" in md assert "`run-abc`" in md and "`run-def`" in md assert "**[低] 响应偏慢**(场景:售前场景)" in md assert "### 分场景叙述" in md assert "**售后场景**:售后场景表现不稳定" in md assert "### 改善建议" in md # 建议按 priority 升序 assert md.index("首要建议") < md.index("次要建议") def test_render_campaign_markdown_analysis_falls_back_to_id_prefix(): md = render_campaign_markdown(_campaign_report(), analysis=_analysis(), scenario_names={}) assert "**[高] 售后答非所问**(场景:s-1)" in md def test_render_campaign_markdown_without_analysis_unchanged(): md = render_campaign_markdown(_campaign_report()) assert "智能分析" not in md def test_render_campaign_markdown_header_readability(): md = render_campaign_markdown(_campaign_report(), target_name="客服机器人") assert "**评测对象**: 客服机器人" in md assert "**状态**: 已完成" in md assert "**窗口**: 2h(正式线)" in md assert "**开始时间**: 2026-07-30 00:00" in md assert "**完成时间**: 2026-07-30 02:00" in md assert "| 0h–1h | 2 | 75.0% | 100.0% | 150ms |" in md def test_render_campaign_markdown_header_falls_back_to_target_id(): md = render_campaign_markdown(_campaign_report()) assert "**评测对象**: t-1" in md def test_render_campaign_markdown_accelerated_line(): report = _campaign_report() report["window_seconds"] = 86400 report["time_scale"] = 4.0 md = render_campaign_markdown(report) assert "**窗口**: 24h(加速调试线 ×4,压缩后实际耗时约 6h)" in md def _comparison() -> dict: """A hand-built dict matching the comparison context the export endpoint resolves.""" return { "result": { "trend": "regressing", "summary": "整体质量下滑,售后场景恶化", "problem_evolution": [ { "status": "persisting", "title": "售后答非所问", "scenario_ids": ["s-1"], "detail": "问题仍未收敛", }, {"status": "resolved", "title": "响应偏慢", "scenario_ids": [], "detail": ""}, ], "suggestion_tracking": [ {"status": "partial", "text": "补充售后知识库", "note": "仅覆盖部分问题"}, {"status": "new", "text": "新增建议:监控时延", "note": ""}, ], }, "baseline_name": "上一期巡检", "baseline_completed_at": "2026-07-29T02:00:00+00:00", "model_name": "qwen-max", "updated_at": "2026-07-30T03:00:00+00:00", "metric_diff": { "overall": { "pass_rate": {"baseline": 0.8, "current": 0.75, "delta": -0.05}, "availability": {"baseline": 1.0, "current": 0.5, "delta": -0.5}, "avg_latency_ms": {"baseline": 120.0, "current": 150.0, "delta": 30.0}, }, "scenarios": [ { "scenario_id": "s-1", "scenario_name": "售后场景", "pass_rate": {"baseline": 0.7, "current": 0.6, "delta": -0.1}, "availability": {"baseline": None, "current": None, "delta": None}, "avg_latency_ms": {"baseline": 100.0, "current": 140.0, "delta": 40.0}, }, ], }, } def test_render_campaign_markdown_appends_comparison_section(): md = render_campaign_markdown( _campaign_report(), comparison=_comparison(), scenario_names={"s-1": "售后场景"}, ) assert "## 周期对比" in md assert "基线:「上一期巡检」(完成于 2026-07-29 02:00)" in md assert "分析模型:qwen-max" in md assert "生成于:2026-07-30 03:00" in md assert "**趋势**:退化 — 整体质量下滑,售后场景恶化" in md assert "### 指标变化" in md assert "| 整窗(总体) | 80.0% → 75.0%(-5.0pp) | 100.0% → 50.0%(-50.0pp) | 120ms → 150ms(+30.0ms) |" in md assert "| 售后场景 | 70.0% → 60.0%(-10.0pp) | — → —(—) | 100ms → 140ms(+40.0ms) |" in md assert "### 问题演变" in md assert "**[持续] 售后答非所问**(场景:售后场景)" in md assert " 问题仍未收敛" in md assert "**[消解] 响应偏慢**" in md assert "### 建议落实情况" in md assert "**[部分落实] 补充售后知识库**" in md assert " 仅覆盖部分问题" in md assert "**[新增] 新增建议:监控时延**" in md def test_render_campaign_markdown_comparison_comes_after_analysis(): md = render_campaign_markdown(_campaign_report(), analysis=_analysis(), comparison=_comparison()) assert md.index("## 智能分析") < md.index("## 周期对比") def test_render_campaign_markdown_without_comparison_unchanged(): md = render_campaign_markdown(_campaign_report()) assert "周期对比" not in md # ── render_campaign_markdown: 探索发现附录(v0.9 票据 05)──────────────────── def _exploration() -> dict: return { "session_count": 3, "sessions_with_experience": 3, "goal_achieved_count": 1, "goal_achievement_rate": 0.3333, "issues": [ {"issue": "缴费入口难找", "count": 2}, {"issue": "验证码收不到", "count": 1}, ], "misled": [{"issue": "被误导选了错误套餐", "count": 1}], "judge_review": { "reviewed_sessions": 1, "findings": [{"dimension": "hallucination", "rating": "poor", "comment": "编造了不存在的政策"}], "summaries": ["服务态度好但存在幻觉"], }, } def test_render_campaign_markdown_appends_exploration_appendix(): md = render_campaign_markdown(_campaign_report(), exploration=_exploration()) assert "## 探索发现" in md assert "| 探索会话数 | 3 |" in md assert "| 目标达成率 | 33.3% |" in md assert "缴费入口难找 ×2" in md assert "验证码收不到 ×1" in md assert "被误导选了错误套餐 ×1" in md assert "编造了不存在的政策" in md assert "复核结论:服务态度好但存在幻觉" in md def test_render_campaign_markdown_exploration_without_judge_or_issues(): md = render_campaign_markdown( _campaign_report(), exploration={ "session_count": 1, "sessions_with_experience": 1, "goal_achieved_count": 1, "goal_achievement_rate": 1.0, "issues": [], "misled": [], "judge_review": None, }, ) assert "## 探索发现" in md assert "无" in md # 空问题清单回落文案 def test_render_campaign_markdown_exploration_comes_after_comparison(): md = render_campaign_markdown( _campaign_report(), analysis=_analysis(), comparison=_comparison(), exploration=_exploration() ) assert md.index("## 智能分析") < md.index("## 周期对比") < md.index("## 探索发现") def test_render_campaign_markdown_without_exploration_byte_identical(): base = render_campaign_markdown(_campaign_report(), analysis=_analysis(), comparison=_comparison()) with_none = render_campaign_markdown( _campaign_report(), analysis=_analysis(), comparison=_comparison(), exploration=None ) assert base == with_none assert "探索发现" not in base