solahqb/AgentEvalTool
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(exploration): findings flow into report, analysis and export

Browse Source 
Exploration sessions aggregate into a single exploration summary
(session counts, goal-achievement rate, issue lists from experience
records, judge conclusions when reviewed) that feeds three exits:
the campaign report gains an exploration dimension, the v0.7 analysis
stage-two input gains the summary (stats only, never full dialogues),
and the Markdown export appends a findings appendix after analysis and
comparison. With no exploration data every output stays unchanged.
This commit is contained in:
sinohqb 2026-08-03 19:16:33 +08:00
parent 9cf64ab0e2
commit 2484c207af
8 changed files with 411 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
backend/agenteval/evaluation/analysis.py
View File

@ -14,6 +14,7 @@ from typing import Any, Awaitable, Callable, Optional
from sqlmodel import Session
from agenteval.evaluation.report import generate_campaign_report
from agenteval.exploration.summary import summarize_exploration
from agenteval.model_gateway import ModelGateway
from agenteval.models import Campaign, ModelCapability, RunStatus
from agenteval.services.model_configs import (
@ -26,6 +27,7 @@ from agenteval.storage.model_config_repository import ModelConfigRepository
from agenteval.storage.repository import (
CampaignAnalysisRepository,
CampaignRepository,
ExplorationSessionRepository,
RunRepository,
ScenarioRepository,
)
@ -152,6 +154,7 @@ async def _synthesize(
report: dict[str, Any],
stage1: list[dict[str, Any]],
chat_client: ChatClient,
exploration_summary: Optional[dict[str, Any]] = None,
) -> dict[str, Any]:
"""阶段二：汇总各场景产出，产总体结论 + 跨场景问题 + 优先级建议。"""
system_prompt = (
@ -163,17 +166,18 @@ async def _synthesize(
'"suggestions": [{"priority": 1, "text": "可执行的改善建议"}]}'
"；问题按严重度从高到低排列，建议按优先级排列。全部使用中文。"
)
user_prompt = json.dumps(
{
"活动": {
"名称": campaign.name,
"窗口秒数": campaign.window_seconds,
"总体指标": report.get("summary", {}),
},
"各场景诊断": stage1,
payload: dict[str, Any] = {
"活动": {
"名称": campaign.name,
"窗口秒数": campaign.window_seconds,
"总体指标": report.get("summary", {}),
},
ensure_ascii=False,
)
"各场景诊断": stage1,
}
if exploration_summary is not None:
# 探索式评测证据线：只给统计与问题清单，不含全量对话
payload["探索发现"] = exploration_summary
user_prompt = json.dumps(payload, ensure_ascii=False)
parsed = _parse_stage(
await chat_client([
{"role": "system", "content": system_prompt},
@ -194,6 +198,7 @@ async def analyze_campaign(
failure_samples: dict[str, list[dict[str, str]]],
valid_run_ids: set[str],
chat_client: ChatClient,
exploration_summary: Optional[dict[str, Any]] = None,
) -> dict[str, Any]:
"""两阶段编排：阶段一按场景并行诊断，阶段二综合研判。
@ -209,7 +214,7 @@ async def analyze_campaign(
_analyze_scenario(entry, failure_samples.get(entry["scenario_id"], []), chat_client)
for entry in capability
])
stage2 = await _synthesize(campaign, report, list(stage1), chat_client)
stage2 = await _synthesize(campaign, report, list(stage1), chat_client, exploration_summary=exploration_summary)
valid_scenario_ids = {entry["scenario_id"] for entry in capability}
problems = []
@ -304,12 +309,14 @@ async def execute_campaign_analysis(
runs = RunRepository(session).list_by_campaign(campaign_id)
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
report = generate_campaign_report(campaign, runs, scenario_names=scenario_names)
exploration_sessions = ExplorationSessionRepository(session).list_by_campaign(campaign_id)
result = await analyze_campaign(
campaign=campaign,
report=report,
failure_samples=collect_failure_samples(campaign_id, session),
valid_run_ids={r.id for r in runs if r.id},
chat_client=client,
exploration_summary=summarize_exploration(exploration_sessions),
)
except Exception as exc:
_logger.warning("活动 %s 智能分析失败: %s", campaign_id, exc)

50
backend/agenteval/evaluation/report_render.py
View File

@ -347,19 +347,63 @@ def _window_line(report: dict[str, Any]) -> str:
return f"**窗口**: {_offset(window)}（加速调试线 ×{scale:g}{wall}）"
def _render_exploration_lines(exploration: dict[str, Any]) -> list[str]:
"""探索发现附录：会话统计 + 问题清单 + judge 复核发现（有才渲染）。"""
lines: list[str] = [
"",
"## 探索发现",
"",
"| 指标 | 数值 |",
"|------|------|",
f"| 探索会话数 | {exploration.get('session_count', 0)} |",
f"| 有体验记录会话数 | {exploration.get('sessions_with_experience', 0)} |",
f"| 目标达成率 | {_pct(exploration.get('goal_achievement_rate'))} |",
"",
"### 问题清单",
"",
]
issues = exploration.get("issues") or []
misled = exploration.get("misled") or []
if not issues and not misled:
lines.append("无")
else:
for item in issues:
lines.append(f"- {item['issue']} ×{item['count']}")
for item in misled:
lines.append(f"- （被误导）{item['issue']} ×{item['count']}")
judge = exploration.get("judge_review")
if judge:
lines += [
"",
"### judge 复核",
"",
f"已复核 {judge.get('reviewed_sessions', 0)} 个会话：",
]
findings = judge.get("findings") or []
if findings:
for item in findings:
lines.append(f"- [{item.get('dimension')}/{item.get('rating')}] {item.get('comment')}")
else:
lines.append("- 未发现问题")
for summary_text in judge.get("summaries") or []:
lines.append(f"- 复核结论：{summary_text}")
return lines
def render_campaign_markdown(
report: dict[str, Any],
*,
analysis: Optional[dict[str, Any]] = None,
comparison: Optional[dict[str, Any]] = None,
exploration: Optional[dict[str, Any]] = None,
target_name: Optional[str] = None,
scenario_names: Optional[dict[str, str]] = None,
) -> str:
"""Render a dual-axis campaign report dict as Markdown.
``analysis`` and ``comparison`` are the stored 智能分析 / 周期对比 results
(completed only); when absent the corresponding appendix is omitted
entirely.
(completed only); ``exploration`` is the on-the-fly 探索发现 aggregate;
when absent the corresponding appendix is omitted entirely (缺则无痕).
"""
s = report["summary"]
status = _CAMPAIGN_STATUS_LABELS.get(report.get("status"), report.get("status") or "-")
@ -409,4 +453,6 @@ def render_campaign_markdown(
lines += _render_analysis_lines(analysis, scenario_names or {})
if comparison:
lines += _render_comparison_lines(comparison, scenario_names or {})
if exploration:
lines += _render_exploration_lines(exploration)
return "\n".join(lines)

73
backend/agenteval/exploration/summary.py Normal file
View File

@ -0,0 +1,73 @@
"""Exploration findings aggregation (探索发现聚合).
把探索会话的体验记录聚合成探索摘要：会话数、目标达成率、问题清单
（blockers / misled 按出现次数降序）、judge 复核结论（已就位才纳入，只收
poor 档发现）。这是体验记录这条第一手证据线进入报告 / 分析 / 导出三个
出口前的唯一聚合口径；只含统计与问题清单，不含全量对话。
"""
from collections import Counter
from typing import Any, Optional
from agenteval.exploration.models import ExplorationSession
_POOR_RATING = "poor"
def _issue_list(counter: Counter) -> list[dict[str, Any]]:
return [{"issue": issue, "count": count} for issue, count in counter.most_common()]
def summarize_exploration(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]:
"""聚合一组探索会话为探索摘要；无会话返回 None（缺则无痕）。"""
if not sessions:
return None
with_experience = [s for s in sessions if s.experience is not None]
blockers: Counter = Counter()
misled: Counter = Counter()
achieved = 0
for session_obj in with_experience:
experience = session_obj.experience
if experience.get("goal_achieved"):
achieved += 1
blockers.update(experience.get("blockers") or [])
misled.update(experience.get("misled") or [])
judge = _summarize_judge_reviews(sessions)
return {
"session_count": len(sessions),
"sessions_with_experience": len(with_experience),
"goal_achieved_count": achieved,
"goal_achievement_rate": round(achieved / len(with_experience), 4) if with_experience else None,
"issues": _issue_list(blockers),
"misled": _issue_list(misled),
"judge_review": judge,
}
def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]:
"""只纳入复核完成的会话；findings 只收 poor 档发现，summaries 收复核总体结论。"""
findings: list[dict[str, Any]] = []
summaries: list[str] = []
reviewed = 0
for session_obj in sessions:
review = session_obj.judge_review
if not review or review.get("status") != "completed":
continue
reviewed += 1
if review.get("summary"):
summaries.append(str(review["summary"]))
for item in review.get("dimensions") or []:
if item.get("rating") == _POOR_RATING:
findings.append(
{
"dimension": item.get("dimension"),
"rating": item.get("rating"),
"comment": item.get("comment") or "",
}
)
if reviewed == 0:
return None
return {"reviewed_sessions": reviewed, "findings": findings, "summaries": summaries}

9
backend/agenteval/web/routers/campaigns.py
View File

@ -26,6 +26,7 @@ from agenteval.evaluation.report import (
summarize_campaign_progress,
)
from agenteval.evaluation.report_render import render_campaign_markdown
from agenteval.exploration.summary import summarize_exploration
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
from agenteval.storage.db import iso_utc, utc_now
from agenteval.storage.model_config_repository import ModelConfigRepository
@ -33,6 +34,7 @@ from agenteval.storage.repository import (
CampaignAnalysisRepository,
CampaignPeriodComparisonRepository,
CampaignRepository,
ExplorationSessionRepository,
RunRepository,
ScenarioRepository,
TargetRepository,
@ -130,7 +132,11 @@ async def get_campaign_report(campaign_id: str, session: Session = Depends(get_d
raise HTTPException(status_code=404, detail="campaign not found")
runs = RunRepository(session).list_by_campaign(campaign_id)
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
return generate_campaign_report(campaign, runs, scenario_names=scenario_names)
report = generate_campaign_report(campaign, runs, scenario_names=scenario_names)
exploration = summarize_exploration(ExplorationSessionRepository(session).list_by_campaign(campaign_id))
if exploration is not None:
report["exploration"] = exploration
return report
@router.get("/{campaign_id}/report/markdown")
@ -174,6 +180,7 @@ async def get_campaign_report_markdown(campaign_id: str, session: Session = Depe
generate_campaign_report(campaign, runs, scenario_names=scenario_names),
analysis=analysis,
comparison=comparison,
exploration=summarize_exploration(ExplorationSessionRepository(session).list_by_campaign(campaign_id)),
target_name=target_name,
scenario_names=scenario_names,
)

55
tests/integration/test_campaigns_api.py
View File

@ -302,6 +302,61 @@ async def test_campaign_report_markdown_export(client, seeded_db):
assert "## 能力汇总" in resp.text
# ── exploration findings in report/export (v0.9 ticket 05) ───────────────────
def _seed_exploration_session(seeded_db, campaign_id: str) -> None:
from agenteval.exploration.models import ExplorationSession
from agenteval.storage.repository import ExplorationSessionRepository
repo = ExplorationSessionRepository(seeded_db)
session_obj = repo.create(
ExplorationSession(campaign_id=campaign_id, target_id="t-1", persona={"name": "x"}, goal="缴费")
)
session_obj.experience = {
"goal_achieved": True,
"blockers": ["缴费入口难找"],
"misled": [],
"emotion": "neutral",
"notes": "",
}
repo.update(session_obj)
async def test_campaign_report_includes_exploration_findings(client, seeded_db):
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
_seed_exploration_session(seeded_db, campaign_id)
report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json()
exploration = report["exploration"]
assert exploration["session_count"] == 1
assert exploration["goal_achievement_rate"] == 1.0
assert exploration["issues"] == [{"issue": "缴费入口难找", "count": 1}]
async def test_campaign_report_without_exploration_omits_key(client, seeded_db):
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json()
assert "exploration" not in report
async def test_markdown_export_appends_exploration_appendix(client, seeded_db):
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
_seed_exploration_session(seeded_db, campaign_id)
resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown")
assert resp.status_code == 200
assert "## 探索发现" in resp.text
assert "缴费入口难找 ×1" in resp.text
async def test_markdown_export_without_exploration_leaves_no_trace(client, seeded_db):
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown")
assert resp.status_code == 200
assert "探索发现" not in resp.text
# ── campaign timeline endpoint (ticket 06) ───────────────────────────────────
async def test_campaign_timeline_structure(client, seeded_db):

39
tests/unit/test_campaign_analysis.py
View File

@ -152,6 +152,45 @@ async def test_scenario_without_failure_samples_still_gets_narrative():
assert len(result["scenario_narratives"]) == 2
async def test_analysis_input_includes_exploration_summary():
"""v0.9 票据 05：阶段二输入追加探索摘要（问题清单 + 达成统计，非全量对话）。"""
exploration_summary = {
"session_count": 2,
"sessions_with_experience": 2,
"goal_achieved_count": 1,
"goal_achievement_rate": 0.5,
"issues": [{"issue": "缴费入口难找", "count": 2}],
"misled": [],
"judge_review": None,
}
client = FakeChatClient(STAGE1_A, STAGE1_B, STAGE2)
await analyze_campaign(
campaign=_campaign(),
report=_report(),
failure_samples={},
valid_run_ids=set(),
chat_client=client,
exploration_summary=exploration_summary,
)
stage2_prompt = json.dumps(client.calls[2], ensure_ascii=False)
assert "探索发现" in stage2_prompt
assert "缴费入口难找" in stage2_prompt
assert "goal_achievement_rate" in stage2_prompt or "0.5" in stage2_prompt
async def test_analysis_without_exploration_omits_section():
client = FakeChatClient(STAGE1_A, STAGE1_B, STAGE2)
await analyze_campaign(
campaign=_campaign(),
report=_report(),
failure_samples={},
valid_run_ids=set(),
chat_client=client,
)
stage2_prompt = json.dumps(client.calls[2], ensure_ascii=False)
assert "探索发现" not in stage2_prompt
# ── 分析模型解析 ─────────────────────────────────────────────────────────
def _seed_config(session, config_id: str, *, analysis_default: bool = False, enabled: bool = True) -> None:

103
tests/unit/test_exploration_summary.py Normal file
View File

@ -0,0 +1,103 @@
"""Exploration summary aggregation (v0.9 票据 05).
体验记录聚合成探索摘要：会话数、目标达成率、问题清单（blockers/misled
按出现次数降序）、judge 复核结论（已就位才纳入，只收 poor 档发现）。
缺则无痕：无会话返回 None。纯函数，不落库。
"""
from agenteval.exploration.models import ExplorationSession, ExplorationSessionStatus
from agenteval.exploration.summary import summarize_exploration
def _session(experience=None, judge_review=None, status=ExplorationSessionStatus.COMPLETED) -> ExplorationSession:
return ExplorationSession(
campaign_id="c-1",
target_id="t-1",
persona={"name": "x"},
goal="查询账单",
status=status,
experience=experience,
judge_review=judge_review,
)
def _experience(goal_achieved: bool, blockers=None, misled=None) -> dict:
return {
"goal_achieved": goal_achieved,
"blockers": blockers or [],
"misled": misled or [],
"emotion": "neutral",
"notes": "",
}
def test_no_sessions_returns_none():
assert summarize_exploration([]) is None
def test_aggregates_counts_rate_and_issue_lists():
sessions = [
_session(_experience(True, blockers=["缴费入口难找"])),
_session(_experience(False, blockers=["缴费入口难找", "验证码收不到"], misled=["被误导选了错误套餐"])),
_session(_experience(False, blockers=["验证码收不到"])),
]
summary = summarize_exploration(sessions)
assert summary["session_count"] == 3
assert summary["sessions_with_experience"] == 3
assert summary["goal_achieved_count"] == 1
assert summary["goal_achievement_rate"] == round(1 / 3, 4)
# 问题清单按出现次数降序（同频按首次出现顺序）
assert summary["issues"] == [
{"issue": "缴费入口难找", "count": 2},
{"issue": "验证码收不到", "count": 2},
]
assert summary["misled"] == [{"issue": "被误导选了错误套餐", "count": 1}]
def test_sessions_without_experience_excluded_from_rate_denominator():
sessions = [
_session(_experience(True)),
_session(), # 未关闭/无体验记录
]
summary = summarize_exploration(sessions)
assert summary["session_count"] == 2
assert summary["sessions_with_experience"] == 1
assert summary["goal_achievement_rate"] == 1.0
assert summary["issues"] == []
def test_no_experience_at_all_rate_is_none():
summary = summarize_exploration([_session(), _session()])
assert summary["goal_achievement_rate"] is None
assert summary["issues"] == [] and summary["misled"] == []
def test_judge_review_included_only_when_completed():
good_review = {
"status": "completed",
"dimensions": [
{"dimension": "attitude", "rating": "good", "comment": "友好"},
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"},
],
"summary": "存在幻觉",
}
failed_review = {"status": "failed", "error": "解析失败"}
sessions = [
_session(_experience(True), judge_review=good_review),
_session(_experience(False), judge_review=failed_review),
_session(_experience(True)), # 未复核
]
summary = summarize_exploration(sessions)
judge = summary["judge_review"]
assert judge["reviewed_sessions"] == 1
# 只收 poor 档发现（问题清单口径）
assert judge["findings"] == [
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"}
]
# 复核总体结论一并纳入
assert judge["summaries"] == ["存在幻觉"]
def test_judge_review_absent_when_none_reviewed():
summary = summarize_exploration([_session(_experience(True))])
assert summary["judge_review"] is None

67
tests/unit/test_report_render.py
View File

@ -346,3 +346,70 @@ def test_render_campaign_markdown_comparison_comes_after_analysis():
def test_render_campaign_markdown_without_comparison_unchanged():
md = render_campaign_markdown(_campaign_report())
assert "周期对比" not in md
# ── render_campaign_markdown: 探索发现附录（v0.9 票据 05）────────────────────
def _exploration() -> dict:
return {
"session_count": 3,
"sessions_with_experience": 3,
"goal_achieved_count": 1,
"goal_achievement_rate": 0.3333,
"issues": [
{"issue": "缴费入口难找", "count": 2},
{"issue": "验证码收不到", "count": 1},
],
"misled": [{"issue": "被误导选了错误套餐", "count": 1}],
"judge_review": {
"reviewed_sessions": 1,
"findings": [{"dimension": "hallucination", "rating": "poor", "comment": "编造了不存在的政策"}],
"summaries": ["服务态度好但存在幻觉"],
},
}
def test_render_campaign_markdown_appends_exploration_appendix():
md = render_campaign_markdown(_campaign_report(), exploration=_exploration())
assert "## 探索发现" in md
assert "| 探索会话数 | 3 |" in md
assert "| 目标达成率 | 33.3% |" in md
assert "缴费入口难找 ×2" in md
assert "验证码收不到 ×1" in md
assert "被误导选了错误套餐 ×1" in md
assert "编造了不存在的政策" in md
assert "复核结论：服务态度好但存在幻觉" in md
def test_render_campaign_markdown_exploration_without_judge_or_issues():
md = render_campaign_markdown(
_campaign_report(),
exploration={
"session_count": 1,
"sessions_with_experience": 1,
"goal_achieved_count": 1,
"goal_achievement_rate": 1.0,
"issues": [],
"misled": [],
"judge_review": None,
},
)
assert "## 探索发现" in md
assert "无" in md # 空问题清单回落文案
def test_render_campaign_markdown_exploration_comes_after_comparison():
md = render_campaign_markdown(
_campaign_report(), analysis=_analysis(), comparison=_comparison(), exploration=_exploration()
)
assert md.index("## 智能分析") < md.index("## 周期对比") < md.index("## 探索发现")
def test_render_campaign_markdown_without_exploration_byte_identical():
base = render_campaign_markdown(_campaign_report(), analysis=_analysis(), comparison=_comparison())
with_none = render_campaign_markdown(
_campaign_report(), analysis=_analysis(), comparison=_comparison(), exploration=None
)
assert base == with_none
assert "探索发现" not in base