2484c207af
Exploration sessions aggregate into a single exploration summary (session counts, goal-achievement rate, issue lists from experience records, judge conclusions when reviewed) that feeds three exits: the campaign report gains an exploration dimension, the v0.7 analysis stage-two input gains the summary (stats only, never full dialogues), and the Markdown export appends a findings appendix after analysis and comparison. With no exploration data every output stays unchanged.
416 lines
15 KiB
Python
416 lines
15 KiB
Python
"""Unit tests for the pure renderers: dict in, HTML/Markdown/JSON out (no DB)."""
|
||
|
||
import json
|
||
|
||
from agenteval.evaluation.report_render import (
|
||
render_campaign_markdown,
|
||
render_html,
|
||
render_json,
|
||
render_markdown,
|
||
)
|
||
|
||
|
||
def _run_report(**overrides) -> dict:
|
||
"""A hand-built report dict matching generate_report's shape."""
|
||
report = {
|
||
"run_id": "run-1",
|
||
"target_id": "t-1",
|
||
"target_name": "客服机器人",
|
||
"scenario_id": "s-1",
|
||
"scenario_name": "售后场景",
|
||
"scenario_version": 2,
|
||
"triggered_by": "manual",
|
||
"status": "completed",
|
||
"started_at": "2026-07-30T10:00:00+00:00",
|
||
"completed_at": "2026-07-30T10:05:00+00:00",
|
||
"summary": {
|
||
"total_cases": 2,
|
||
"passed_cases": 1,
|
||
"failed_cases": 1,
|
||
"total_rules": 3,
|
||
"passed_rules": 2,
|
||
"pass_rate": 0.5,
|
||
"connectivity_cases": 1,
|
||
"judged_pass_rate": 0.0,
|
||
},
|
||
"cases": [
|
||
{
|
||
"case_id": "case-a",
|
||
"passed": True,
|
||
"connectivity": True,
|
||
"turns": [
|
||
{
|
||
"round": 0,
|
||
"sent_text": "你好",
|
||
"reply_text": "您好，请问有什么可以帮您？",
|
||
"latency_ms": 120,
|
||
"question_msg_id": "m-1",
|
||
}
|
||
],
|
||
"results": [],
|
||
},
|
||
{
|
||
"case_id": "case-b",
|
||
"passed": False,
|
||
"connectivity": False,
|
||
"turns": [
|
||
{
|
||
"round": 0,
|
||
"sent_text": "退货流程",
|
||
"reply_text": None,
|
||
"latency_ms": None,
|
||
"question_msg_id": "m-2",
|
||
}
|
||
],
|
||
"results": [
|
||
{"rule_type": "keyword_match", "passed": False, "score": 0.0, "reason": "缺少关键词"},
|
||
],
|
||
},
|
||
],
|
||
}
|
||
report.update(overrides)
|
||
return report
|
||
|
||
|
||
def _campaign_report() -> dict:
|
||
"""A hand-built dict matching generate_campaign_report's shape."""
|
||
return {
|
||
"campaign_id": "c-1",
|
||
"name": "夜间巡检",
|
||
"target_id": "t-1",
|
||
"status": "completed",
|
||
"window_seconds": 7200,
|
||
"time_scale": 1.0,
|
||
"started_at": "2026-07-30T00:00:00+00:00",
|
||
"completed_at": "2026-07-30T02:00:00+00:00",
|
||
"summary": {
|
||
"total_runs": 2,
|
||
"completed_runs": 2,
|
||
"overall_pass_rate": 0.75,
|
||
"overall_availability": 1.0,
|
||
"avg_latency_ms": 150.0,
|
||
},
|
||
"time_trend": [
|
||
{
|
||
"bucket_index": 0,
|
||
"start_seconds": 0.0,
|
||
"end_seconds": 3600.0,
|
||
"run_count": 2,
|
||
"pass_rate": 0.75,
|
||
"availability": 1.0,
|
||
"avg_latency_ms": 150.0,
|
||
},
|
||
{
|
||
"bucket_index": 1,
|
||
"start_seconds": 3600.0,
|
||
"end_seconds": 7200.0,
|
||
"run_count": 0,
|
||
"pass_rate": None,
|
||
"availability": None,
|
||
"avg_latency_ms": None,
|
||
},
|
||
],
|
||
"capability_summary": [
|
||
{
|
||
"scenario_id": "s-1",
|
||
"scenario_name": "售后场景",
|
||
"run_count": 2,
|
||
"pass_rate": 0.75,
|
||
"availability": 1.0,
|
||
"avg_latency_ms": 150.0,
|
||
},
|
||
],
|
||
}
|
||
|
||
|
||
# ── render_html ─────────────────────────────────────────────────────────────
|
||
|
||
def test_render_html_contains_names_and_summary():
|
||
html = render_html(_run_report())
|
||
assert "客服机器人" in html
|
||
assert "售后场景" in html
|
||
assert "50.00%" in html # pass_rate 0.5
|
||
|
||
|
||
def test_render_html_contains_turns_and_rule_badges():
|
||
html = render_html(_run_report())
|
||
assert "退货流程" in html
|
||
assert "keyword_match" in html
|
||
assert "失败" in html
|
||
|
||
|
||
# ── render_markdown ─────────────────────────────────────────────────────────
|
||
|
||
def test_render_markdown_summary_table():
|
||
md = render_markdown(_run_report())
|
||
assert "| 总用例数 | 2 |" in md
|
||
assert "| 通过率 | 50.0% |" in md
|
||
assert "| 连通用例 | 1 |" in md
|
||
|
||
|
||
def test_render_markdown_connectivity_badge_and_no_reply():
|
||
md = render_markdown(_run_report())
|
||
assert "🔗" in md # connectivity case badge
|
||
assert "（连通用例，未配置判定标准）" in md
|
||
assert "（无回复）" in md
|
||
|
||
|
||
def test_render_markdown_judged_pass_rate_dash_when_none():
|
||
report = _run_report()
|
||
report["summary"]["judged_pass_rate"] = None
|
||
md = render_markdown(report)
|
||
assert "| 判定型通过率 | — |" in md
|
||
|
||
|
||
# ── render_json ─────────────────────────────────────────────────────────────
|
||
|
||
def test_render_json_roundtrips():
|
||
report = _run_report()
|
||
parsed = json.loads(render_json(report))
|
||
assert parsed == report
|
||
|
||
|
||
# ── render_campaign_markdown ────────────────────────────────────────────────
|
||
|
||
def test_render_campaign_markdown_summary_and_axes():
|
||
md = render_campaign_markdown(_campaign_report())
|
||
assert "# 活动周期报告 — 夜间巡检" in md
|
||
assert "| 整窗通过率 | 75.0% |" in md
|
||
assert "## 时间趋势" in md
|
||
assert "## 能力汇总" in md
|
||
assert "| 售后场景 | 2 | 75.0% | 100.0% | 150ms |" in md
|
||
|
||
|
||
def test_render_campaign_markdown_empty_bucket_dashes():
|
||
md = render_campaign_markdown(_campaign_report())
|
||
# bucket 1 has no runs: pass_rate/availability/latency all render as —
|
||
assert "| 1h–2h | 0 | — | — | — |" in md
|
||
|
||
|
||
def _analysis() -> dict:
|
||
"""A hand-built dict matching the stored campaign analysis result shape."""
|
||
return {
|
||
"overall": "整窗通过率偏低，售后场景拖后腿",
|
||
"problems": [
|
||
{
|
||
"severity": "high",
|
||
"title": "售后答非所问",
|
||
"description": "多轮对话中反复偏离用户问题",
|
||
"scenario_ids": ["s-1"],
|
||
"evidence_run_ids": ["run-abc", "run-def"],
|
||
},
|
||
{
|
||
"severity": "low",
|
||
"title": "响应偏慢",
|
||
"description": "高峰时段时延偏高",
|
||
"scenario_ids": ["s-2"],
|
||
"evidence_run_ids": [],
|
||
},
|
||
],
|
||
"scenario_narratives": [
|
||
{"scenario_id": "s-1", "narrative": "售后场景表现不稳定"},
|
||
{"scenario_id": "s-2", "narrative": "售前场景表现稳定"},
|
||
],
|
||
"suggestions": [
|
||
{"priority": 2, "text": "次要建议：扩容"},
|
||
{"priority": 1, "text": "首要建议：补充售后知识库"},
|
||
],
|
||
}
|
||
|
||
|
||
def test_render_campaign_markdown_appends_analysis_sections():
|
||
md = render_campaign_markdown(
|
||
_campaign_report(),
|
||
analysis=_analysis(),
|
||
scenario_names={"s-1": "售后场景", "s-2": "售前场景"},
|
||
)
|
||
assert "## 智能分析" in md
|
||
assert "### 总体结论" in md
|
||
assert "整窗通过率偏低，售后场景拖后腿" in md
|
||
assert "### 问题诊断" in md
|
||
assert "**[高] 售后答非所问**（场景：售后场景）" in md
|
||
assert "`run-abc`" in md and "`run-def`" in md
|
||
assert "**[低] 响应偏慢**（场景：售前场景）" in md
|
||
assert "### 分场景叙述" in md
|
||
assert "**售后场景**：售后场景表现不稳定" in md
|
||
assert "### 改善建议" in md
|
||
# 建议按 priority 升序
|
||
assert md.index("首要建议") < md.index("次要建议")
|
||
|
||
|
||
def test_render_campaign_markdown_analysis_falls_back_to_id_prefix():
|
||
md = render_campaign_markdown(_campaign_report(), analysis=_analysis(), scenario_names={})
|
||
assert "**[高] 售后答非所问**（场景：s-1）" in md
|
||
|
||
|
||
def test_render_campaign_markdown_without_analysis_unchanged():
|
||
md = render_campaign_markdown(_campaign_report())
|
||
assert "智能分析" not in md
|
||
|
||
|
||
def test_render_campaign_markdown_header_readability():
|
||
md = render_campaign_markdown(_campaign_report(), target_name="客服机器人")
|
||
assert "**评测对象**: 客服机器人" in md
|
||
assert "**状态**: 已完成" in md
|
||
assert "**窗口**: 2h（正式线）" in md
|
||
assert "**开始时间**: 2026-07-30 00:00" in md
|
||
assert "**完成时间**: 2026-07-30 02:00" in md
|
||
assert "| 0h–1h | 2 | 75.0% | 100.0% | 150ms |" in md
|
||
|
||
|
||
def test_render_campaign_markdown_header_falls_back_to_target_id():
|
||
md = render_campaign_markdown(_campaign_report())
|
||
assert "**评测对象**: t-1" in md
|
||
|
||
|
||
def test_render_campaign_markdown_accelerated_line():
|
||
report = _campaign_report()
|
||
report["window_seconds"] = 86400
|
||
report["time_scale"] = 4.0
|
||
md = render_campaign_markdown(report)
|
||
assert "**窗口**: 24h（加速调试线 ×4，压缩后实际耗时约 6h）" in md
|
||
|
||
|
||
def _comparison() -> dict:
|
||
"""A hand-built dict matching the comparison context the export endpoint resolves."""
|
||
return {
|
||
"result": {
|
||
"trend": "regressing",
|
||
"summary": "整体质量下滑，售后场景恶化",
|
||
"problem_evolution": [
|
||
{
|
||
"status": "persisting",
|
||
"title": "售后答非所问",
|
||
"scenario_ids": ["s-1"],
|
||
"detail": "问题仍未收敛",
|
||
},
|
||
{"status": "resolved", "title": "响应偏慢", "scenario_ids": [], "detail": ""},
|
||
],
|
||
"suggestion_tracking": [
|
||
{"status": "partial", "text": "补充售后知识库", "note": "仅覆盖部分问题"},
|
||
{"status": "new", "text": "新增建议：监控时延", "note": ""},
|
||
],
|
||
},
|
||
"baseline_name": "上一期巡检",
|
||
"baseline_completed_at": "2026-07-29T02:00:00+00:00",
|
||
"model_name": "qwen-max",
|
||
"updated_at": "2026-07-30T03:00:00+00:00",
|
||
"metric_diff": {
|
||
"overall": {
|
||
"pass_rate": {"baseline": 0.8, "current": 0.75, "delta": -0.05},
|
||
"availability": {"baseline": 1.0, "current": 0.5, "delta": -0.5},
|
||
"avg_latency_ms": {"baseline": 120.0, "current": 150.0, "delta": 30.0},
|
||
},
|
||
"scenarios": [
|
||
{
|
||
"scenario_id": "s-1",
|
||
"scenario_name": "售后场景",
|
||
"pass_rate": {"baseline": 0.7, "current": 0.6, "delta": -0.1},
|
||
"availability": {"baseline": None, "current": None, "delta": None},
|
||
"avg_latency_ms": {"baseline": 100.0, "current": 140.0, "delta": 40.0},
|
||
},
|
||
],
|
||
},
|
||
}
|
||
|
||
|
||
def test_render_campaign_markdown_appends_comparison_section():
|
||
md = render_campaign_markdown(
|
||
_campaign_report(),
|
||
comparison=_comparison(),
|
||
scenario_names={"s-1": "售后场景"},
|
||
)
|
||
assert "## 周期对比" in md
|
||
assert "基线：「上一期巡检」（完成于 2026-07-29 02:00）" in md
|
||
assert "分析模型：qwen-max" in md
|
||
assert "生成于：2026-07-30 03:00" in md
|
||
assert "**趋势**：退化 — 整体质量下滑，售后场景恶化" in md
|
||
assert "### 指标变化" in md
|
||
assert "| 整窗（总体） | 80.0% → 75.0%（-5.0pp） | 100.0% → 50.0%（-50.0pp） | 120ms → 150ms（+30.0ms） |" in md
|
||
assert "| 售后场景 | 70.0% → 60.0%（-10.0pp） | — → —（—） | 100ms → 140ms（+40.0ms） |" in md
|
||
assert "### 问题演变" in md
|
||
assert "**[持续] 售后答非所问**（场景：售后场景）" in md
|
||
assert " 问题仍未收敛" in md
|
||
assert "**[消解] 响应偏慢**" in md
|
||
assert "### 建议落实情况" in md
|
||
assert "**[部分落实] 补充售后知识库**" in md
|
||
assert " 仅覆盖部分问题" in md
|
||
assert "**[新增] 新增建议：监控时延**" in md
|
||
|
||
|
||
def test_render_campaign_markdown_comparison_comes_after_analysis():
|
||
md = render_campaign_markdown(_campaign_report(), analysis=_analysis(), comparison=_comparison())
|
||
assert md.index("## 智能分析") < md.index("## 周期对比")
|
||
|
||
|
||
def test_render_campaign_markdown_without_comparison_unchanged():
|
||
md = render_campaign_markdown(_campaign_report())
|
||
assert "周期对比" not in md
|
||
|
||
|
||
# ── render_campaign_markdown: 探索发现附录（v0.9 票据 05）────────────────────
|
||
|
||
|
||
def _exploration() -> dict:
|
||
return {
|
||
"session_count": 3,
|
||
"sessions_with_experience": 3,
|
||
"goal_achieved_count": 1,
|
||
"goal_achievement_rate": 0.3333,
|
||
"issues": [
|
||
{"issue": "缴费入口难找", "count": 2},
|
||
{"issue": "验证码收不到", "count": 1},
|
||
],
|
||
"misled": [{"issue": "被误导选了错误套餐", "count": 1}],
|
||
"judge_review": {
|
||
"reviewed_sessions": 1,
|
||
"findings": [{"dimension": "hallucination", "rating": "poor", "comment": "编造了不存在的政策"}],
|
||
"summaries": ["服务态度好但存在幻觉"],
|
||
},
|
||
}
|
||
|
||
|
||
def test_render_campaign_markdown_appends_exploration_appendix():
|
||
md = render_campaign_markdown(_campaign_report(), exploration=_exploration())
|
||
assert "## 探索发现" in md
|
||
assert "| 探索会话数 | 3 |" in md
|
||
assert "| 目标达成率 | 33.3% |" in md
|
||
assert "缴费入口难找 ×2" in md
|
||
assert "验证码收不到 ×1" in md
|
||
assert "被误导选了错误套餐 ×1" in md
|
||
assert "编造了不存在的政策" in md
|
||
assert "复核结论：服务态度好但存在幻觉" in md
|
||
|
||
|
||
def test_render_campaign_markdown_exploration_without_judge_or_issues():
|
||
md = render_campaign_markdown(
|
||
_campaign_report(),
|
||
exploration={
|
||
"session_count": 1,
|
||
"sessions_with_experience": 1,
|
||
"goal_achieved_count": 1,
|
||
"goal_achievement_rate": 1.0,
|
||
"issues": [],
|
||
"misled": [],
|
||
"judge_review": None,
|
||
},
|
||
)
|
||
assert "## 探索发现" in md
|
||
assert "无" in md # 空问题清单回落文案
|
||
|
||
|
||
def test_render_campaign_markdown_exploration_comes_after_comparison():
|
||
md = render_campaign_markdown(
|
||
_campaign_report(), analysis=_analysis(), comparison=_comparison(), exploration=_exploration()
|
||
)
|
||
assert md.index("## 智能分析") < md.index("## 周期对比") < md.index("## 探索发现")
|
||
|
||
|
||
def test_render_campaign_markdown_without_exploration_byte_identical():
|
||
base = render_campaign_markdown(_campaign_report(), analysis=_analysis(), comparison=_comparison())
|
||
with_none = render_campaign_markdown(
|
||
_campaign_report(), analysis=_analysis(), comparison=_comparison(), exploration=None
|
||
)
|
||
assert base == with_none
|
||
assert "探索发现" not in base
|