diff --git a/backend/agenteval/evaluation/analysis.py b/backend/agenteval/evaluation/analysis.py index a8cf63d..c88e23a 100644 --- a/backend/agenteval/evaluation/analysis.py +++ b/backend/agenteval/evaluation/analysis.py @@ -14,6 +14,7 @@ from typing import Any, Awaitable, Callable, Optional from sqlmodel import Session from agenteval.evaluation.report import generate_campaign_report +from agenteval.exploration.summary import summarize_exploration from agenteval.model_gateway import ModelGateway from agenteval.models import Campaign, ModelCapability, RunStatus from agenteval.services.model_configs import ( @@ -26,6 +27,7 @@ from agenteval.storage.model_config_repository import ModelConfigRepository from agenteval.storage.repository import ( CampaignAnalysisRepository, CampaignRepository, + ExplorationSessionRepository, RunRepository, ScenarioRepository, ) @@ -152,6 +154,7 @@ async def _synthesize( report: dict[str, Any], stage1: list[dict[str, Any]], chat_client: ChatClient, + exploration_summary: Optional[dict[str, Any]] = None, ) -> dict[str, Any]: """阶段二:汇总各场景产出,产总体结论 + 跨场景问题 + 优先级建议。""" system_prompt = ( @@ -163,17 +166,18 @@ async def _synthesize( '"suggestions": [{"priority": 1, "text": "可执行的改善建议"}]}' ";问题按严重度从高到低排列,建议按优先级排列。全部使用中文。" ) - user_prompt = json.dumps( - { - "活动": { - "名称": campaign.name, - "窗口秒数": campaign.window_seconds, - "总体指标": report.get("summary", {}), - }, - "各场景诊断": stage1, + payload: dict[str, Any] = { + "活动": { + "名称": campaign.name, + "窗口秒数": campaign.window_seconds, + "总体指标": report.get("summary", {}), }, - ensure_ascii=False, - ) + "各场景诊断": stage1, + } + if exploration_summary is not None: + # 探索式评测证据线:只给统计与问题清单,不含全量对话 + payload["探索发现"] = exploration_summary + user_prompt = json.dumps(payload, ensure_ascii=False) parsed = _parse_stage( await chat_client([ {"role": "system", "content": system_prompt}, @@ -194,6 +198,7 @@ async def analyze_campaign( failure_samples: dict[str, list[dict[str, str]]], valid_run_ids: set[str], chat_client: ChatClient, + exploration_summary: Optional[dict[str, Any]] = None, ) -> dict[str, Any]: """两阶段编排:阶段一按场景并行诊断,阶段二综合研判。 @@ -209,7 +214,7 @@ async def analyze_campaign( _analyze_scenario(entry, failure_samples.get(entry["scenario_id"], []), chat_client) for entry in capability ]) - stage2 = await _synthesize(campaign, report, list(stage1), chat_client) + stage2 = await _synthesize(campaign, report, list(stage1), chat_client, exploration_summary=exploration_summary) valid_scenario_ids = {entry["scenario_id"] for entry in capability} problems = [] @@ -304,12 +309,14 @@ async def execute_campaign_analysis( runs = RunRepository(session).list_by_campaign(campaign_id) scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()} report = generate_campaign_report(campaign, runs, scenario_names=scenario_names) + exploration_sessions = ExplorationSessionRepository(session).list_by_campaign(campaign_id) result = await analyze_campaign( campaign=campaign, report=report, failure_samples=collect_failure_samples(campaign_id, session), valid_run_ids={r.id for r in runs if r.id}, chat_client=client, + exploration_summary=summarize_exploration(exploration_sessions), ) except Exception as exc: _logger.warning("活动 %s 智能分析失败: %s", campaign_id, exc) diff --git a/backend/agenteval/evaluation/report_render.py b/backend/agenteval/evaluation/report_render.py index 88a5762..5c5616b 100644 --- a/backend/agenteval/evaluation/report_render.py +++ b/backend/agenteval/evaluation/report_render.py @@ -347,19 +347,63 @@ def _window_line(report: dict[str, Any]) -> str: return f"**窗口**: {_offset(window)}(加速调试线 ×{scale:g}{wall})" +def _render_exploration_lines(exploration: dict[str, Any]) -> list[str]: + """探索发现附录:会话统计 + 问题清单 + judge 复核发现(有才渲染)。""" + lines: list[str] = [ + "", + "## 探索发现", + "", + "| 指标 | 数值 |", + "|------|------|", + f"| 探索会话数 | {exploration.get('session_count', 0)} |", + f"| 有体验记录会话数 | {exploration.get('sessions_with_experience', 0)} |", + f"| 目标达成率 | {_pct(exploration.get('goal_achievement_rate'))} |", + "", + "### 问题清单", + "", + ] + issues = exploration.get("issues") or [] + misled = exploration.get("misled") or [] + if not issues and not misled: + lines.append("无") + else: + for item in issues: + lines.append(f"- {item['issue']} ×{item['count']}") + for item in misled: + lines.append(f"- (被误导){item['issue']} ×{item['count']}") + judge = exploration.get("judge_review") + if judge: + lines += [ + "", + "### judge 复核", + "", + f"已复核 {judge.get('reviewed_sessions', 0)} 个会话:", + ] + findings = judge.get("findings") or [] + if findings: + for item in findings: + lines.append(f"- [{item.get('dimension')}/{item.get('rating')}] {item.get('comment')}") + else: + lines.append("- 未发现问题") + for summary_text in judge.get("summaries") or []: + lines.append(f"- 复核结论:{summary_text}") + return lines + + def render_campaign_markdown( report: dict[str, Any], *, analysis: Optional[dict[str, Any]] = None, comparison: Optional[dict[str, Any]] = None, + exploration: Optional[dict[str, Any]] = None, target_name: Optional[str] = None, scenario_names: Optional[dict[str, str]] = None, ) -> str: """Render a dual-axis campaign report dict as Markdown. ``analysis`` and ``comparison`` are the stored 智能分析 / 周期对比 results - (completed only); when absent the corresponding appendix is omitted - entirely. + (completed only); ``exploration`` is the on-the-fly 探索发现 aggregate; + when absent the corresponding appendix is omitted entirely (缺则无痕). """ s = report["summary"] status = _CAMPAIGN_STATUS_LABELS.get(report.get("status"), report.get("status") or "-") @@ -409,4 +453,6 @@ def render_campaign_markdown( lines += _render_analysis_lines(analysis, scenario_names or {}) if comparison: lines += _render_comparison_lines(comparison, scenario_names or {}) + if exploration: + lines += _render_exploration_lines(exploration) return "\n".join(lines) diff --git a/backend/agenteval/exploration/summary.py b/backend/agenteval/exploration/summary.py new file mode 100644 index 0000000..94347af --- /dev/null +++ b/backend/agenteval/exploration/summary.py @@ -0,0 +1,73 @@ +"""Exploration findings aggregation (探索发现聚合). + +把探索会话的体验记录聚合成探索摘要:会话数、目标达成率、问题清单 +(blockers / misled 按出现次数降序)、judge 复核结论(已就位才纳入,只收 +poor 档发现)。这是体验记录这条第一手证据线进入报告 / 分析 / 导出三个 +出口前的唯一聚合口径;只含统计与问题清单,不含全量对话。 +""" + +from collections import Counter +from typing import Any, Optional + +from agenteval.exploration.models import ExplorationSession + +_POOR_RATING = "poor" + + +def _issue_list(counter: Counter) -> list[dict[str, Any]]: + return [{"issue": issue, "count": count} for issue, count in counter.most_common()] + + +def summarize_exploration(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]: + """聚合一组探索会话为探索摘要;无会话返回 None(缺则无痕)。""" + if not sessions: + return None + + with_experience = [s for s in sessions if s.experience is not None] + blockers: Counter = Counter() + misled: Counter = Counter() + achieved = 0 + for session_obj in with_experience: + experience = session_obj.experience + if experience.get("goal_achieved"): + achieved += 1 + blockers.update(experience.get("blockers") or []) + misled.update(experience.get("misled") or []) + + judge = _summarize_judge_reviews(sessions) + + return { + "session_count": len(sessions), + "sessions_with_experience": len(with_experience), + "goal_achieved_count": achieved, + "goal_achievement_rate": round(achieved / len(with_experience), 4) if with_experience else None, + "issues": _issue_list(blockers), + "misled": _issue_list(misled), + "judge_review": judge, + } + + +def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]: + """只纳入复核完成的会话;findings 只收 poor 档发现,summaries 收复核总体结论。""" + findings: list[dict[str, Any]] = [] + summaries: list[str] = [] + reviewed = 0 + for session_obj in sessions: + review = session_obj.judge_review + if not review or review.get("status") != "completed": + continue + reviewed += 1 + if review.get("summary"): + summaries.append(str(review["summary"])) + for item in review.get("dimensions") or []: + if item.get("rating") == _POOR_RATING: + findings.append( + { + "dimension": item.get("dimension"), + "rating": item.get("rating"), + "comment": item.get("comment") or "", + } + ) + if reviewed == 0: + return None + return {"reviewed_sessions": reviewed, "findings": findings, "summaries": summaries} diff --git a/backend/agenteval/web/routers/campaigns.py b/backend/agenteval/web/routers/campaigns.py index 6c9c491..802fdf0 100644 --- a/backend/agenteval/web/routers/campaigns.py +++ b/backend/agenteval/web/routers/campaigns.py @@ -26,6 +26,7 @@ from agenteval.evaluation.report import ( summarize_campaign_progress, ) from agenteval.evaluation.report_render import render_campaign_markdown +from agenteval.exploration.summary import summarize_exploration from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds from agenteval.storage.db import iso_utc, utc_now from agenteval.storage.model_config_repository import ModelConfigRepository @@ -33,6 +34,7 @@ from agenteval.storage.repository import ( CampaignAnalysisRepository, CampaignPeriodComparisonRepository, CampaignRepository, + ExplorationSessionRepository, RunRepository, ScenarioRepository, TargetRepository, @@ -130,7 +132,11 @@ async def get_campaign_report(campaign_id: str, session: Session = Depends(get_d raise HTTPException(status_code=404, detail="campaign not found") runs = RunRepository(session).list_by_campaign(campaign_id) scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()} - return generate_campaign_report(campaign, runs, scenario_names=scenario_names) + report = generate_campaign_report(campaign, runs, scenario_names=scenario_names) + exploration = summarize_exploration(ExplorationSessionRepository(session).list_by_campaign(campaign_id)) + if exploration is not None: + report["exploration"] = exploration + return report @router.get("/{campaign_id}/report/markdown") @@ -174,6 +180,7 @@ async def get_campaign_report_markdown(campaign_id: str, session: Session = Depe generate_campaign_report(campaign, runs, scenario_names=scenario_names), analysis=analysis, comparison=comparison, + exploration=summarize_exploration(ExplorationSessionRepository(session).list_by_campaign(campaign_id)), target_name=target_name, scenario_names=scenario_names, ) diff --git a/tests/integration/test_campaigns_api.py b/tests/integration/test_campaigns_api.py index bc89c26..22892e0 100644 --- a/tests/integration/test_campaigns_api.py +++ b/tests/integration/test_campaigns_api.py @@ -302,6 +302,61 @@ async def test_campaign_report_markdown_export(client, seeded_db): assert "## 能力汇总" in resp.text +# ── exploration findings in report/export (v0.9 ticket 05) ─────────────────── + + +def _seed_exploration_session(seeded_db, campaign_id: str) -> None: + from agenteval.exploration.models import ExplorationSession + from agenteval.storage.repository import ExplorationSessionRepository + + repo = ExplorationSessionRepository(seeded_db) + session_obj = repo.create( + ExplorationSession(campaign_id=campaign_id, target_id="t-1", persona={"name": "x"}, goal="缴费") + ) + session_obj.experience = { + "goal_achieved": True, + "blockers": ["缴费入口难找"], + "misled": [], + "emotion": "neutral", + "notes": "", + } + repo.update(session_obj) + + +async def test_campaign_report_includes_exploration_findings(client, seeded_db): + campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] + _seed_exploration_session(seeded_db, campaign_id) + + report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json() + exploration = report["exploration"] + assert exploration["session_count"] == 1 + assert exploration["goal_achievement_rate"] == 1.0 + assert exploration["issues"] == [{"issue": "缴费入口难找", "count": 1}] + + +async def test_campaign_report_without_exploration_omits_key(client, seeded_db): + campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] + report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json() + assert "exploration" not in report + + +async def test_markdown_export_appends_exploration_appendix(client, seeded_db): + campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] + _seed_exploration_session(seeded_db, campaign_id) + + resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown") + assert resp.status_code == 200 + assert "## 探索发现" in resp.text + assert "缴费入口难找 ×1" in resp.text + + +async def test_markdown_export_without_exploration_leaves_no_trace(client, seeded_db): + campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] + resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown") + assert resp.status_code == 200 + assert "探索发现" not in resp.text + + # ── campaign timeline endpoint (ticket 06) ─────────────────────────────────── async def test_campaign_timeline_structure(client, seeded_db): diff --git a/tests/unit/test_campaign_analysis.py b/tests/unit/test_campaign_analysis.py index 1a62441..3375083 100644 --- a/tests/unit/test_campaign_analysis.py +++ b/tests/unit/test_campaign_analysis.py @@ -152,6 +152,45 @@ async def test_scenario_without_failure_samples_still_gets_narrative(): assert len(result["scenario_narratives"]) == 2 +async def test_analysis_input_includes_exploration_summary(): + """v0.9 票据 05:阶段二输入追加探索摘要(问题清单 + 达成统计,非全量对话)。""" + exploration_summary = { + "session_count": 2, + "sessions_with_experience": 2, + "goal_achieved_count": 1, + "goal_achievement_rate": 0.5, + "issues": [{"issue": "缴费入口难找", "count": 2}], + "misled": [], + "judge_review": None, + } + client = FakeChatClient(STAGE1_A, STAGE1_B, STAGE2) + await analyze_campaign( + campaign=_campaign(), + report=_report(), + failure_samples={}, + valid_run_ids=set(), + chat_client=client, + exploration_summary=exploration_summary, + ) + stage2_prompt = json.dumps(client.calls[2], ensure_ascii=False) + assert "探索发现" in stage2_prompt + assert "缴费入口难找" in stage2_prompt + assert "goal_achievement_rate" in stage2_prompt or "0.5" in stage2_prompt + + +async def test_analysis_without_exploration_omits_section(): + client = FakeChatClient(STAGE1_A, STAGE1_B, STAGE2) + await analyze_campaign( + campaign=_campaign(), + report=_report(), + failure_samples={}, + valid_run_ids=set(), + chat_client=client, + ) + stage2_prompt = json.dumps(client.calls[2], ensure_ascii=False) + assert "探索发现" not in stage2_prompt + + # ── 分析模型解析 ───────────────────────────────────────────────────────── def _seed_config(session, config_id: str, *, analysis_default: bool = False, enabled: bool = True) -> None: diff --git a/tests/unit/test_exploration_summary.py b/tests/unit/test_exploration_summary.py new file mode 100644 index 0000000..29fb8bd --- /dev/null +++ b/tests/unit/test_exploration_summary.py @@ -0,0 +1,103 @@ +"""Exploration summary aggregation (v0.9 票据 05). + +体验记录聚合成探索摘要:会话数、目标达成率、问题清单(blockers/misled +按出现次数降序)、judge 复核结论(已就位才纳入,只收 poor 档发现)。 +缺则无痕:无会话返回 None。纯函数,不落库。 +""" + +from agenteval.exploration.models import ExplorationSession, ExplorationSessionStatus +from agenteval.exploration.summary import summarize_exploration + + +def _session(experience=None, judge_review=None, status=ExplorationSessionStatus.COMPLETED) -> ExplorationSession: + return ExplorationSession( + campaign_id="c-1", + target_id="t-1", + persona={"name": "x"}, + goal="查询账单", + status=status, + experience=experience, + judge_review=judge_review, + ) + + +def _experience(goal_achieved: bool, blockers=None, misled=None) -> dict: + return { + "goal_achieved": goal_achieved, + "blockers": blockers or [], + "misled": misled or [], + "emotion": "neutral", + "notes": "", + } + + +def test_no_sessions_returns_none(): + assert summarize_exploration([]) is None + + +def test_aggregates_counts_rate_and_issue_lists(): + sessions = [ + _session(_experience(True, blockers=["缴费入口难找"])), + _session(_experience(False, blockers=["缴费入口难找", "验证码收不到"], misled=["被误导选了错误套餐"])), + _session(_experience(False, blockers=["验证码收不到"])), + ] + summary = summarize_exploration(sessions) + assert summary["session_count"] == 3 + assert summary["sessions_with_experience"] == 3 + assert summary["goal_achieved_count"] == 1 + assert summary["goal_achievement_rate"] == round(1 / 3, 4) + # 问题清单按出现次数降序(同频按首次出现顺序) + assert summary["issues"] == [ + {"issue": "缴费入口难找", "count": 2}, + {"issue": "验证码收不到", "count": 2}, + ] + assert summary["misled"] == [{"issue": "被误导选了错误套餐", "count": 1}] + + +def test_sessions_without_experience_excluded_from_rate_denominator(): + sessions = [ + _session(_experience(True)), + _session(), # 未关闭/无体验记录 + ] + summary = summarize_exploration(sessions) + assert summary["session_count"] == 2 + assert summary["sessions_with_experience"] == 1 + assert summary["goal_achievement_rate"] == 1.0 + assert summary["issues"] == [] + + +def test_no_experience_at_all_rate_is_none(): + summary = summarize_exploration([_session(), _session()]) + assert summary["goal_achievement_rate"] is None + assert summary["issues"] == [] and summary["misled"] == [] + + +def test_judge_review_included_only_when_completed(): + good_review = { + "status": "completed", + "dimensions": [ + {"dimension": "attitude", "rating": "good", "comment": "友好"}, + {"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"}, + ], + "summary": "存在幻觉", + } + failed_review = {"status": "failed", "error": "解析失败"} + sessions = [ + _session(_experience(True), judge_review=good_review), + _session(_experience(False), judge_review=failed_review), + _session(_experience(True)), # 未复核 + ] + summary = summarize_exploration(sessions) + judge = summary["judge_review"] + assert judge["reviewed_sessions"] == 1 + # 只收 poor 档发现(问题清单口径) + assert judge["findings"] == [ + {"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"} + ] + # 复核总体结论一并纳入 + assert judge["summaries"] == ["存在幻觉"] + + +def test_judge_review_absent_when_none_reviewed(): + summary = summarize_exploration([_session(_experience(True))]) + assert summary["judge_review"] is None diff --git a/tests/unit/test_report_render.py b/tests/unit/test_report_render.py index 9619cc5..aacc411 100644 --- a/tests/unit/test_report_render.py +++ b/tests/unit/test_report_render.py @@ -346,3 +346,70 @@ def test_render_campaign_markdown_comparison_comes_after_analysis(): def test_render_campaign_markdown_without_comparison_unchanged(): md = render_campaign_markdown(_campaign_report()) assert "周期对比" not in md + + +# ── render_campaign_markdown: 探索发现附录(v0.9 票据 05)──────────────────── + + +def _exploration() -> dict: + return { + "session_count": 3, + "sessions_with_experience": 3, + "goal_achieved_count": 1, + "goal_achievement_rate": 0.3333, + "issues": [ + {"issue": "缴费入口难找", "count": 2}, + {"issue": "验证码收不到", "count": 1}, + ], + "misled": [{"issue": "被误导选了错误套餐", "count": 1}], + "judge_review": { + "reviewed_sessions": 1, + "findings": [{"dimension": "hallucination", "rating": "poor", "comment": "编造了不存在的政策"}], + "summaries": ["服务态度好但存在幻觉"], + }, + } + + +def test_render_campaign_markdown_appends_exploration_appendix(): + md = render_campaign_markdown(_campaign_report(), exploration=_exploration()) + assert "## 探索发现" in md + assert "| 探索会话数 | 3 |" in md + assert "| 目标达成率 | 33.3% |" in md + assert "缴费入口难找 ×2" in md + assert "验证码收不到 ×1" in md + assert "被误导选了错误套餐 ×1" in md + assert "编造了不存在的政策" in md + assert "复核结论:服务态度好但存在幻觉" in md + + +def test_render_campaign_markdown_exploration_without_judge_or_issues(): + md = render_campaign_markdown( + _campaign_report(), + exploration={ + "session_count": 1, + "sessions_with_experience": 1, + "goal_achieved_count": 1, + "goal_achievement_rate": 1.0, + "issues": [], + "misled": [], + "judge_review": None, + }, + ) + assert "## 探索发现" in md + assert "无" in md # 空问题清单回落文案 + + +def test_render_campaign_markdown_exploration_comes_after_comparison(): + md = render_campaign_markdown( + _campaign_report(), analysis=_analysis(), comparison=_comparison(), exploration=_exploration() + ) + assert md.index("## 智能分析") < md.index("## 周期对比") < md.index("## 探索发现") + + +def test_render_campaign_markdown_without_exploration_byte_identical(): + base = render_campaign_markdown(_campaign_report(), analysis=_analysis(), comparison=_comparison()) + with_none = render_campaign_markdown( + _campaign_report(), analysis=_analysis(), comparison=_comparison(), exploration=None + ) + assert base == with_none + assert "探索发现" not in base