From 1e55a21649d2d8e0bf01ccd94ddb6e6a68a15718 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Mon, 3 Aug 2026 13:32:48 +0800 Subject: [PATCH] =?UTF-8?q?feat(comparison):=20v0.8=20=E5=91=A8=E6=9C=9F?= =?UTF-8?q?=E5=AF=B9=E6=AF=94=20=E2=80=94=20=E8=AE=A1=E5=88=92=E6=8C=87?= =?UTF-8?q?=E7=BA=B9=E8=87=AA=E5=8A=A8=E5=9F=BA=E7=BA=BF=E9=85=8D=E5=AF=B9?= =?UTF-8?q?=E3=80=81=E6=9C=BA=E6=A2=B0=E6=8C=87=E6=A0=87=20diff=20?= =?UTF-8?q?=E4=B8=8E=20LLM=20=E6=BC=94=E8=BF=9B=E5=8F=99=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 正式线活动分析完成后自动链式生成对比；报告抽屉新增「周期对比」区块 （趋势徽章、指标 delta 表、问题演变、建议追踪，无自动基线时手动选择）。 版本号升至 0.8.0。 --- .../01-auto-baseline-and-metric-diff.md | 13 + .../02-comparison-narrative-agent-and-api.md | 13 + .../03-report-drawer-comparison-section.md | 16 + .../issues/04-auto-chain-after-analysis.md | 12 + .scratch/v0.8/spec.md | 94 ++++ CONTEXT.md | 8 + backend/agenteval/evaluation/analysis.py | 24 + backend/agenteval/evaluation/comparison.py | 419 ++++++++++++++++ backend/agenteval/storage/db.py | 27 ++ backend/agenteval/storage/repository.py | 47 ++ backend/agenteval/web/routers/campaigns.py | 105 +++- frontend/web/package-lock.json | 4 +- frontend/web/package.json | 2 +- frontend/web/src/api.ts | 74 +++ frontend/web/src/pages/Campaigns.tsx | 281 ++++++++++- ...7c34d18_add_campaign_period_comparisons.py | 51 ++ pyproject.toml | 2 +- .../test_campaign_comparison_api.py | 409 ++++++++++++++++ .../test_campaign_comparison_auto_trigger.py | 171 +++++++ tests/unit/test_campaign_comparison.py | 448 ++++++++++++++++++ ...t_campaign_period_comparisons_migration.py | 31 ++ 21 files changed, 2244 insertions(+), 7 deletions(-) create mode 100644 .scratch/v0.8/issues/01-auto-baseline-and-metric-diff.md create mode 100644 .scratch/v0.8/issues/02-comparison-narrative-agent-and-api.md create mode 100644 .scratch/v0.8/issues/03-report-drawer-comparison-section.md create mode 100644 .scratch/v0.8/issues/04-auto-chain-after-analysis.md create mode 100644 .scratch/v0.8/spec.md create mode 100644 backend/agenteval/evaluation/comparison.py create mode 100644 migrations/versions/f2a9b7c34d18_add_campaign_period_comparisons.py create mode 100644 tests/integration/test_campaign_comparison_api.py create mode 100644 tests/integration/test_campaign_comparison_auto_trigger.py create mode 100644 tests/unit/test_campaign_comparison.py create mode 100644 tests/unit/test_campaign_period_comparisons_migration.py diff --git a/.scratch/v0.8/issues/01-auto-baseline-and-metric-diff.md b/.scratch/v0.8/issues/01-auto-baseline-and-metric-diff.md new file mode 100644 index 0000000..caf1759 --- /dev/null +++ b/.scratch/v0.8/issues/01-auto-baseline-and-metric-diff.md @@ -0,0 +1,13 @@ +# 01 — 自动基线配对 + 机械指标 diff（GET 先行） + +**What to build:** 周期对比的确定性半边。计划指纹纯函数（对象 + 计划条目集合[场景/偏移/次数] + 窗口，条目顺序无关）；自动基线解析：双方均正式线、指纹相等、完成时间早于本期、已有 completed 分析，取最近一期，无则 None。机械 diff 纯函数：两期 `generate_campaign_report` dict 进，整窗 + 分场景（按 scenario_id 并集）的通过率/可用性/时延 delta 出，任一侧 None 则 delta None。`GET /api/campaigns/{id}/comparison` 先行返回 `{auto_baseline: {id, name, completed_at} | null, metric_diff: {...} | null}`（自动基线存在才算 diff），404 活动不存在。 + +**Blocked by:** 无 — 可立即开始 + +**Status:** done + +- [x] 计划指纹纯函数：同计划不同条目顺序相等；改窗口/对象/场景/偏移/次数任一不等 +- [x] 自动基线：指纹相等取最近一期；跳过无分析/分析非 completed/加速线候选；本期为加速线时返回 None；无匹配返回 None +- [x] 机械 diff：整窗与分场景 delta 数值正确；并集场景缺一侧 delta None；None 指标传播 +- [x] GET 返回自动基线 + 现算 diff；无基线时两个 null；404 +- [x] `pytest` + `ruff` 全绿 diff --git a/.scratch/v0.8/issues/02-comparison-narrative-agent-and-api.md b/.scratch/v0.8/issues/02-comparison-narrative-agent-and-api.md new file mode 100644 index 0000000..04fce71 --- /dev/null +++ b/.scratch/v0.8/issues/02-comparison-narrative-agent-and-api.md @@ -0,0 +1,13 @@ +# 02 — 对比叙述生成（表 + Agent + POST） + +**What to build:** 周期对比的 LLM 半边与完整生命周期。新表 `campaign_period_comparisons`（当前活动 id 唯一 upsert、baseline_campaign_id、generating/completed/failed、result JSON、model_config_id 快照、error、triggered_by、时间戳）+ 迁移。单次 LLM 调用编排：输入两期分析 JSON + 机械 diff，复用 `resolve_analysis_model` 与 ChatClient 注入 seam；输出 schema 见 spec（trend/summary/problem_evolution/suggestion_tracking）；scenario_ids 白名单剔除；非法枚举归一（trend→stable、演化→persisting、建议→unaddressed）；解析失败落 failed。`POST /api/campaigns/{id}/comparison`：非终态 400；本期无 completed 分析 400；body 可带 `baseline_campaign_id`（缺省走自动基线，仍无 → 400 引导；手动基线须有 completed 分析）；无模型 400 引导；成功 → 后台任务 + `{"status": "generating"}`。GET 扩展合并：状态行字段 + 机械 diff + 叙述 + 基线信息。 + +**Blocked by:** 01（GET 与配对复用） + +**Status:** done + +- [x] 迁移：`campaign_period_comparisons` 表（campaign_id 唯一约束）+ 迁移测试 +- [x] 仓储 upsert / get_by_campaign +- [x] 编排：FakeChatClient 断言输入含两期分析与 diff；解析失败→failed；白名单剔除虚构 scenario_ids；非法枚举归一 +- [x] POST 各 400 分支 + 成功分支 + 重跑 upsert 不新增行；GET 合并返回 +- [x] `pytest` + `ruff` 全绿 diff --git a/.scratch/v0.8/issues/03-report-drawer-comparison-section.md b/.scratch/v0.8/issues/03-report-drawer-comparison-section.md new file mode 100644 index 0000000..be0b332 --- /dev/null +++ b/.scratch/v0.8/issues/03-report-drawer-comparison-section.md @@ -0,0 +1,16 @@ +# 03 — 报告抽屉「周期对比」区块 + +**What to build:** 报告抽屉在「智能分析」之下新增「周期对比」区块。状态行：生成中（spinner + 5s 轮询）、失败（错误原因 + 重试）、未生成（「生成对比」按钮；无自动基线时显示基线选择器——同对象历史终态活动下拉，选定后生成）。已生成依次渲染：趋势徽章（improving 绿 / stable 蓝 / regressing 红）+ 总结 → 指标变化表（整窗行 + 分场景行，delta 正绿负红、None 显示 —）→ 问题演化列表（新增红 / 持续橙 / 消解绿 Tag + 标题 + 说明 + 场景 Tag）→ 建议追踪（已落实绿 / 部分橙 / 未落实灰 / 新增蓝）。展示基线活动名与完成时间、所用分析模型名与生成时间。生成按钮仅终态活动可用。 + +**Blocked by:** 02（API 与结果结构） + +**Status:** done + +- [x] `api.ts`：对比类型 + `getComparison` / `generateComparison(id, baselineId?)` +- [x] 区块按状态渲染（生成中轮询复用 `usePolling`） +- [x] 已生成渲染：趋势徽章 + 总结 / 指标 delta 表（正绿负红）/ 问题演化 Tag / 建议追踪状态 +- [x] 无自动基线时的基线选择器（同对象终态活动下拉） +- [x] 基线活动名 + 完成时间 + 模型名 + 生成时间展示；按钮终态门控 +- [x] `npx tsc --noEmit` + `npm run build` 通过；浏览器实操验证 + （注：本会话无浏览器工具，以 `curl` 对 8010 端口本地后端做了 GET 端点冒烟： + 无基线活动返回 `{"status":"none",...}`、未知活动 404；交互式浏览器验证待用户本地确认） diff --git a/.scratch/v0.8/issues/04-auto-chain-after-analysis.md b/.scratch/v0.8/issues/04-auto-chain-after-analysis.md new file mode 100644 index 0000000..bf39412 --- /dev/null +++ b/.scratch/v0.8/issues/04-auto-chain-after-analysis.md @@ -0,0 +1,12 @@ +# 04 — 正式线自动链（分析完成 → 对比生成） + +**What to build:** `execute_campaign_analysis` 成功落 completed 后，活动为正式线且存在自动基线（含 completed 分析）→ 自动 enqueue 对比生成（`triggered_by=auto`）；加速线、无自动基线、无分析模型均静默跳过，不影响分析任务本身。注意：analysis 模块与 comparison 模块互相依赖（对比复用 `resolve_analysis_model`），用函数内延迟导入避免循环 import。 + +**Blocked by:** 02（生成入口） + +**Status:** done + +- [x] 分析完成后自动链：正式线 + 自动基线 → spy 断言 enqueue（`triggered_by=auto`） +- [x] 加速线 / 无自动基线 / 无模型三分支静默跳过 +- [x] 自动链异常不冒泡（警告日志），分析结果不受影响 +- [x] `pytest` + `ruff` 全绿 diff --git a/.scratch/v0.8/spec.md b/.scratch/v0.8/spec.md new file mode 100644 index 0000000..55ca81c --- /dev/null +++ b/.scratch/v0.8/spec.md @@ -0,0 +1,94 @@ +# v0.8 — 周期对比（Period Comparison） + +## Problem Statement + +运营者用评估活动对同一对象做周期性考察，每期末尾拿到一份智能分析（总体结论、问题诊断、改善建议）。但单期分析是孤立的：这一期的问题到底是新冒出来的、持续未愈的、还是已经消解的？上一期的改善建议落实了吗？要回答这些，人得自己打开两期报告逐条对照自由文本——而这正是平台该替人做的元层研判。 + +## Solution + +在相邻两期活动之间提供**周期对比**：机械 diff 给出两期聚合指标的客观变化（整窗与分场景的通过率/可用性/时延 delta，ADR-0004 口径，确定性计算），LLM 在两期分析结论之上产出结构化演进叙述（趋势判断、问题演化、建议追踪）。报告抽屉内新增「周期对比」区块，与智能分析并列。正式线活动全链自动（活动完成 → 分析 → 对比），加速调试线手动按需。 + +## User Stories + +1. 作为运营者，我希望打开本期活动报告时看到"与上一期相比"的指标变化表（整窗通过率/可用性/时延及分场景 delta），以便快速判断走势。 +2. 作为运营者，我希望看到一个趋势徽章（改善/平稳/退化），以便不看数字也能感知方向。 +3. 作为运营者，我希望看到问题演化列表（新增/持续/消解，含涉及场景与说明），以便知道该担心什么、可以放下什么。 +4. 作为运营者，我希望看到上期建议的追踪状态（已落实/部分落实/未落实/新增），以便评估改进动作的有效性。 +5. 作为运营者，我希望正式线活动的对比在分析完成后自动生成，以便零操作拿到结论。 +6. 作为调试者，我希望加速线活动能手动点「生成对比」，以便验证功能而不等完整周期。 +7. 作为运营者，当本期活动计划与历史不一致（无自动基线）时，我希望能手动挑选一个历史活动作基线，以便换计划后仍能对比。 +8. 作为运营者，我希望对比生成失败时看到原因并可重试，以便区分模型问题与数据问题。 +9. 作为运营者，我希望对比叙述与指标 diff 分开呈现（叙述是 LLM 判断、数字是确定性计算），以便知道哪部分可以较真。 +10. 作为管理员，我希望对比复用「分析默认」模型（活动覆盖 ?? 全局默认），以便不引入新的配置面。 +11. 作为运营者，我希望对比中引用的场景是真实存在的（白名单过滤），以便不被模型虚构误导。 + +## Implementation Decisions + +### 配对（计划指纹 + 自动基线） + +- **计划指纹** = 评测对象 + 计划条目集合（场景 id、偏移秒数、次数）+ 窗口秒数。指纹相等的正式线活动构成活动串 +- **自动基线** = 同活动串中、完成时间早于本期、且已有 completed 分析的最近一期活动 +- 指纹不一致 → 无自动基线；`POST /comparison` 接受可选 `baseline_campaign_id` 手动指定（跨串允许，可比性由用户负责） +- 基线活动被删或分析被重置 → 视为无基线 + +### 产出（混合：机械 diff + LLM 叙述） + +- **机械 diff**：纯函数，输入两期 `generate_campaign_report` 结果，输出整窗 + 分场景的通过率/可用性/时延 delta（读时现算，不存储） +- **LLM 叙述**：单次调用（不做两阶段——输入只有两份已消化的分析 JSON + 机械 diff，无原始样本），输入上限轻量；输出结构化 JSON： + +```json +{ + "trend": "improving | stable | regressing", + "summary": "总体演进结论", + "problem_evolution": [{"status": "new | persisting | resolved", "title", "detail", "scenario_ids": []}], + "suggestion_tracking": [{"text", "status": "addressed | partial | unaddressed | new", "note"}] +} +``` + +- 叙述中的 `scenario_ids` 落库前过白名单（同 v0.7）；非法 `trend`/`status` 枚举归一（trend→stable，status→persisting / unaddressed） +- LLM 解析失败 → 行落 failed + error + +### 存储与生命周期 + +- 新表 `campaign_period_comparisons`：当前活动 id 唯一 upsert、baseline_campaign_id、generating/completed/failed、result（叙述 JSON）、model_config_id 快照、error、triggered_by（auto/manual）、时间戳——镜像 `campaign_analyses` 模式 +- 模型解析复用 v0.7 `resolve_analysis_model`（活动覆盖 ?? 全局分析默认）；不可解析 → 自动链静默跳过 / 手动 400 引导 +- 自动链：`execute_campaign_analysis` 成功落 completed 后，活动为正式线且存在自动基线 → enqueue 对比生成；任一前提不满足静默跳过 +- 重新生成（手动 POST）= 同活动 upsert 覆盖，可借此更换基线重算 + +### API + +- `GET /api/campaigns/{id}/comparison`：无行 → 空态 `{"status": "none"}`；有行 → 机械 diff（现算）+ 叙述行合并返回；附 `baseline_campaign_id` 与自动配对结果，前端据此渲染或引导选基线 +- `POST /api/campaigns/{id}/comparison`：body 可选 `{"baseline_campaign_id": "..."}`；非终态活动 400；当前活动无 completed 分析 400（先生成分析）；无模型 400 引导；成功 → 后台任务 + `{"status": "generating"}` +- 两期都有 completed 分析是生成前提（基线无分析 → 400 提示先生成基线分析） + +### 前端（报告抽屉「周期对比」区块） + +- 位置：「智能分析」区块之下，复用区块状态行范式（生成中 5s 轮询 / 失败重试 / 未生成按钮） +- 已生成：趋势徽章 + 总结 → 指标变化表（整窗 + 分场景 delta，正绿负红）→ 问题演化（新增红/持续橙/消解绿 Tag）→ 建议追踪（已落实绿/部分橙/未落实灰/新增蓝） +- 无自动基线：基线选择器（同对象历史终态活动下拉）+ 生成按钮 +- 展示基线活动名与完成时间、所用分析模型名与生成时间 + +## Testing Decisions + +好测试只测外部行为，不测实现细节；全部落在既有最高层 seam： + +1. **机械 diff 纯函数**：手搓两份报告 dict，断言 delta 结构与数值、空桶/None 处理（先例如 `test_report_render.py`） +2. **计划指纹与自动基线**：内存 SQLite，指纹相等/不等、跳过无分析活动、跳过非正式线、取最近一期（先例如 `test_campaign_analysis.py` 的仓储测试） +3. **LLM 叙述编排**：FakeChatClient（v0.7 seam）覆盖成功编排、解析失败→AnalysisError 式失败、白名单剔除虚构 scenario_ids、非法枚举归一 +4. **API 集成**：镜像 `test_campaign_analysis_api.py`——GET 空态、POST 非终态 400、无分析 400、无基线 400、手动指定基线成功、重跑 upsert 不新增行 +5. **自动链**：spy 对比入口，断言分析完成后正式线自动触发、加速线不触发、无基线不触发（镜像 `test_campaign_analysis_auto_trigger.py`） +6. **迁移测试**：表存在 + campaign_id 唯一约束（先例如 `test_campaign_analyses_migration.py`） + +## Out of Scope + +- 跨活动对比（多对象横切）——词汇表中另一个轴，本规格不涉及 +- 活动列表趋势标记（↑↓）——后续增强 +- 对比结果纳入 Markdown 导出——看使用情况再定 +- 基线活动的选取 UI 优化（如按指纹分组展示） +- 三期以上的趋势序列（只对比相邻两期） + +## Further Notes + +- 设计决策源自 `/grill-with-docs` 访谈（6 问 6 决）：配对 A、产出 C 混合、呈现 A 抽屉区块、生命周期 A 新表+联动触发、输入 A 轻量、输出 A 结构化 +- 「周期对比」「正式线/加速调试线」已入 `CONTEXT.md` 词汇表 +- 对比不重新消化失败样本：样本已被两期分析消化，对比是元层研判 diff --git a/CONTEXT.md b/CONTEXT.md index 53f7d92..ff80875 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -74,6 +74,14 @@ _Avoid_: 排程、日程表 活动周期报告中的一个维度：某时段（或整窗）内**正常完成**的子 Run 占比（completed / 已派生）。与通过率正交——通过率反映"回答质量"（用例级、含故障判不通过，ADR-0002），可用性反映"服务是否可达/执行是否成功"。通道故障导致的失败子 Run 拉低可用性。 _Avoid_: 在线率、健康度 +**周期对比（Period Comparison）**: +同一活动串（同评测对象 + 同计划指纹：场景集合、偏移、次数、窗口完全一致）中，相邻两期正式线活动的对照，回答"这一期比上一期好了还是坏了"。自动按完成时间取前一个已有 completed 分析的活动为基线；计划指纹不一致即无自动基线，可手动另选。区别于"跨活动对比"（多对象并行活动的横切对照）与"对比报告"（Run 级逐项对照）。 +_Avoid_: 环比（暗示固定自然周期，活动窗口可任意长） + +**正式线 / 加速调试线（Production-line / Accelerated-line）**: +活动的两种运行档位，由时间倍速区分：`time_scale == 1` 为正式线（窗口按真实时长走完，代表真实服务周期，享受自动分析、自动周期对比等全链自动化）；`time_scale > 1` 为加速调试线（压缩窗口用于快速验证流程，自动化能力默认关闭，仅手动按需触发）。 +_Avoid_: 真实线、快速模式 + ## 模型配置 **模型能力（Capability）**: diff --git a/backend/agenteval/evaluation/analysis.py b/backend/agenteval/evaluation/analysis.py index 79c036d..073135f 100644 --- a/backend/agenteval/evaluation/analysis.py +++ b/backend/agenteval/evaluation/analysis.py @@ -239,6 +239,29 @@ async def analyze_campaign( } +def _maybe_enqueue_period_comparison(campaign: Campaign, session: Session) -> None: + """正式线活动分析完成后自动链到周期对比（v0.8）。 + + 前提：正式线（time_scale == 1）、分析模型可解析、存在自动基线 + （同活动串且已有 completed 分析）。任一不满足静默跳过；异常仅告警， + 不影响刚落库的分析结果。 + """ + try: + if campaign.time_scale != 1: + return + if resolve_analysis_model(campaign, session) is None: + return + # 延迟导入：comparison 顶层依赖 analysis（resolve_analysis_model）， + # 反向导入会成环。 + from agenteval.evaluation import comparison as comparison_module + + if comparison_module.resolve_auto_baseline(campaign, session) is None: + return + comparison_module.start_campaign_comparison(campaign.id, triggered_by="auto") + except Exception as exc: + _logger.warning("活动 %s 自动周期对比跳过: %s", campaign.id, exc) + + def _gateway_chat_client(runtime: ModelRuntimeConfig) -> ChatClient: gateway = ModelGateway(timeout=180.0) @@ -298,6 +321,7 @@ async def execute_campaign_analysis( campaign_id, status="completed", result=result, model_config_id=runtime.id, triggered_by=triggered_by, ) + _maybe_enqueue_period_comparison(campaign, session) finally: session.close() diff --git a/backend/agenteval/evaluation/comparison.py b/backend/agenteval/evaluation/comparison.py new file mode 100644 index 0000000..c94b933 --- /dev/null +++ b/backend/agenteval/evaluation/comparison.py @@ -0,0 +1,419 @@ +"""Period comparison — baseline pairing, metric diff and LLM narrative (周期对比). + +机械半边：计划指纹（同活动串的判据）、自动基线解析、两期报告的指标 +diff——全部确定性计算（ADR-0004 口径，经 ``generate_campaign_report`` 的 +既有聚合结果现算）。叙述半边：单次 LLM 调用，在两期已消化的分析 JSON 与 +机械 diff 之上产出结构化演进叙述（CONTEXT.md「周期对比」）。 +""" + +import asyncio +import json +import logging +from datetime import datetime, timezone +from typing import Any, Optional + +from sqlmodel import Session + +from agenteval.evaluation.analysis import ChatClient, resolve_analysis_model +from agenteval.evaluation.report import generate_campaign_report +from agenteval.model_gateway import ModelGateway +from agenteval.models import Campaign, EvalRun +from agenteval.services.model_configs import ModelRuntimeConfig +from agenteval.storage.db import get_session, iso_utc, utc_now +from agenteval.storage.repository import ( + CampaignAnalysisRepository, + CampaignPeriodComparisonRepository, + CampaignRepository, + RunRepository, + ScenarioRepository, +) +from agenteval.utils.llm import parse_json_from_llm_text + +_logger = logging.getLogger("agenteval") + +_SAME_MOMENT_EPS = 1e-3 + +_VALID_TRENDS = {"improving", "stable", "regressing"} +_VALID_EVOLUTION_STATUSES = {"new", "persisting", "resolved"} +_VALID_SUGGESTION_STATUSES = {"addressed", "partial", "unaddressed", "new"} + + +class ComparisonError(RuntimeError): + """对比生成失败（数据缺失或模型输出无法解析），可重试。""" + + +def campaign_plan_fingerprint(campaign: Campaign) -> str: + """活动串判据：评测对象 + 计划条目集合 + 窗口秒数（条目顺序无关）。 + + 指纹相等的活动构成同一活动串（CONTEXT.md「周期对比」）。档位 + （time_scale）不参与指纹——正式/加速由基线解析另行过滤。 + """ + entries = sorted( + ((entry.scenario_id, entry.offset_seconds, entry.count) for entry in campaign.plan), + key=lambda e: (e[0], e[1], e[2]), + ) + return json.dumps( + {"target": campaign.target_id, "window": campaign.window_seconds, "plan": entries}, + separators=(",", ":"), + ) + + +def _to_utc(dt: Optional[datetime]) -> Optional[datetime]: + if dt is None: + return None + return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt + + +def resolve_auto_baseline(campaign: Campaign, session: Session) -> Optional[Campaign]: + """自动基线：同活动串中最近一期已完成的正式线活动（须有 completed 分析）。 + + 条件（全部满足才可作基线）：指纹相等、双方均为正式线 + （time_scale == 1）、候选完成时间早于本期、候选已有 completed 分析。 + 本期为加速线或无匹配候选时返回 None。 + """ + if campaign.time_scale != 1: + return None + reference = _to_utc(campaign.completed_at) or utc_now() + fingerprint = campaign_plan_fingerprint(campaign) + + best: Optional[Campaign] = None + best_completed: Optional[datetime] = None + for candidate in CampaignRepository(session).list_all(): + if candidate.id == campaign.id or candidate.time_scale != 1: + continue + if campaign_plan_fingerprint(candidate) != fingerprint: + continue + completed = _to_utc(candidate.completed_at) + if completed is None: + continue + if (reference - completed).total_seconds() < _SAME_MOMENT_EPS: + continue + analysis_row = CampaignAnalysisRepository(session).get_by_campaign(candidate.id) + if analysis_row is None or analysis_row.status != "completed": + continue + if best_completed is None or completed > best_completed: + best = candidate + best_completed = completed + return best + + +def _delta(base: Optional[float], current: Optional[float], ndigits: int) -> Optional[float]: + if base is None or current is None: + return None + return round(current - base, ndigits) + + +def compute_metric_diff( + baseline_report: dict[str, Any], + current_report: dict[str, Any], +) -> dict[str, Any]: + """两期 ``generate_campaign_report`` 结果 → 整窗 + 分场景指标 delta。 + + 通过率/可用性按 4 位小数、时延按 1 位小数取 delta；任一侧指标为 None + 或该侧缺失该场景时 delta 为 None（只保留存在的 baseline/current 值）。 + 纯函数，无 I/O。 + """ + + def _pair(base: Optional[float], current: Optional[float], ndigits: int) -> dict[str, Optional[float]]: + return {"baseline": base, "current": current, "delta": _delta(base, current, ndigits)} + + base_summary = baseline_report.get("summary") or {} + cur_summary = current_report.get("summary") or {} + overall = { + "pass_rate": _pair(base_summary.get("overall_pass_rate"), cur_summary.get("overall_pass_rate"), 4), + "availability": _pair(base_summary.get("overall_availability"), cur_summary.get("overall_availability"), 4), + "avg_latency_ms": _pair(base_summary.get("avg_latency_ms"), cur_summary.get("avg_latency_ms"), 1), + } + + base_scenarios = {s.get("scenario_id"): s for s in baseline_report.get("capability_summary") or []} + cur_scenarios = {s.get("scenario_id"): s for s in current_report.get("capability_summary") or []} + + scenarios = [] + for sid in sorted(set(base_scenarios) | set(cur_scenarios), key=lambda x: x or ""): + base = base_scenarios.get(sid) + cur = cur_scenarios.get(sid) + entry: dict[str, Any] = { + "scenario_id": sid, + "scenario_name": (cur or base or {}).get("scenario_name") or "", + "pass_rate": _pair( + base.get("pass_rate") if base else None, + cur.get("pass_rate") if cur else None, + 4, + ), + "availability": _pair( + base.get("availability") if base else None, + cur.get("availability") if cur else None, + 4, + ), + "avg_latency_ms": _pair( + base.get("avg_latency_ms") if base else None, + cur.get("avg_latency_ms") if cur else None, + 1, + ), + } + scenarios.append(entry) + + return {"overall": overall, "scenarios": scenarios} + + +def build_campaign_report_dict(campaign: Campaign, session: Session) -> dict[str, Any]: + """为周期对比现算一期活动的报告 dict（读路径，不重算聚合口径）。""" + runs: list[EvalRun] = RunRepository(session).list_by_campaign(campaign.id) + scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()} + return generate_campaign_report(campaign, runs, scenario_names=scenario_names) + + +def build_comparison_payload(campaign: Campaign, session: Session) -> dict[str, Any]: + """GET 返回体：自动基线信息 + 机械 diff（无基线时两者均为 null）。""" + baseline = resolve_auto_baseline(campaign, session) + if baseline is None: + return {"auto_baseline": None, "metric_diff": None} + diff = compute_metric_diff( + build_campaign_report_dict(baseline, session), + build_campaign_report_dict(campaign, session), + ) + return { + "auto_baseline": { + "id": baseline.id, + "name": baseline.name, + "completed_at": iso_utc(baseline.completed_at), + }, + "metric_diff": diff, + } + + +# ── 叙述半边：单次 LLM 调用编排 ───────────────────────────────────────── + + +async def narrate_period_comparison( + *, + baseline_analysis: dict[str, Any], + current_analysis: dict[str, Any], + metric_diff: dict[str, Any], + valid_scenario_ids: set[str], + chat_client: ChatClient, +) -> dict[str, Any]: + """在两期分析结论与机械 diff 之上产出结构化演进叙述（单次调用）。 + + 输出 schema：trend / summary / problem_evolution / suggestion_tracking。 + 模型虚构的 scenario_id 按白名单剔除；非法枚举归一（trend→stable、 + 演化 status→persisting、建议 status→unaddressed）；解析失败抛 + ``ComparisonError``（由调用方落 failed 状态）。 + """ + system_prompt = ( + "你是智能客服质量评估平台的趋势研判专家，负责对照相邻两期评估活动的" + "智能分析结论，产出结构化的演进叙述。只输出一个 JSON 对象：" + '{"trend": "improving|stable|regressing", ' + '"summary": "总体演进结论（一段话）", ' + '"problem_evolution": [{"status": "new|persisting|resolved", "title": "...", ' + '"detail": "该问题的演化说明", "scenario_ids": ["涉及场景 id"]}], ' + '"suggestion_tracking": [{"text": "上期建议原文", ' + '"status": "addressed|partial|unaddressed|new", "note": "落实情况的说明"}]}' + "；problem_evolution 覆盖上期问题在本期的新增/持续/消解，" + "suggestion_tracking 追踪上期每条建议的落实情况并列出本期新增建议。" + "全部使用中文。" + ) + user_prompt = json.dumps( + { + "上期分析": baseline_analysis, + "本期分析": current_analysis, + "指标变化（确定性计算，可信）": metric_diff, + }, + ensure_ascii=False, + ) + content = await chat_client( + [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ] + ) + try: + parsed = parse_json_from_llm_text(content) + except Exception as exc: + raise ComparisonError(f"对比叙述输出解析失败: {exc}") from exc + if not isinstance(parsed, dict): + raise ComparisonError("对比叙述输出不是 JSON 对象") + + summary = parsed.get("summary") + if not isinstance(summary, str) or not summary.strip(): + raise ComparisonError("对比叙述缺少 summary") + + trend = parsed.get("trend") + if trend not in _VALID_TRENDS: + trend = "stable" + + problem_evolution = [] + for item in parsed.get("problem_evolution") or []: + if not isinstance(item, dict): + continue + status = item.get("status") + problem_evolution.append( + { + "status": status if status in _VALID_EVOLUTION_STATUSES else "persisting", + "title": str(item.get("title", "")), + "detail": str(item.get("detail", "")), + "scenario_ids": [s for s in item.get("scenario_ids") or [] if s in valid_scenario_ids], + } + ) + + suggestion_tracking = [] + for item in parsed.get("suggestion_tracking") or []: + if not isinstance(item, dict): + continue + status = item.get("status") + suggestion_tracking.append( + { + "text": str(item.get("text", "")), + "status": status if status in _VALID_SUGGESTION_STATUSES else "unaddressed", + "note": str(item.get("note", "")), + } + ) + + return { + "trend": trend, + "summary": summary, + "problem_evolution": problem_evolution, + "suggestion_tracking": suggestion_tracking, + } + + +def _gateway_chat_client(runtime: ModelRuntimeConfig) -> ChatClient: + gateway = ModelGateway(timeout=180.0) + + async def _chat(messages: list[dict[str, str]]) -> str: + return await gateway.chat(runtime, messages, temperature=0.2) + + return _chat + + +async def execute_campaign_comparison( + campaign_id: str, + *, + triggered_by: str, + baseline_campaign_id: Optional[str] = None, + chat_client: Optional[ChatClient] = None, +) -> None: + """后台执行体：generating → completed/failed 状态机（upsert，每活动一行）。 + + 基线优先取显式指定（手动重选），缺省走自动基线。与 Runs 同款后台任务 + 约定：自持 Session、try/finally 关闭、失败落 error。 + """ + session = get_session() + try: + comparisons = CampaignPeriodComparisonRepository(session) + analyses = CampaignAnalysisRepository(session) + campaign = CampaignRepository(session).get(campaign_id) + if not campaign: + return + + runtime = resolve_analysis_model(campaign, session) + if runtime is None: + comparisons.upsert( + campaign_id, + status="failed", + triggered_by=triggered_by, + error="未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」", + ) + return + + if baseline_campaign_id: + baseline = CampaignRepository(session).get(baseline_campaign_id) + if baseline is None: + comparisons.upsert( + campaign_id, + status="failed", + triggered_by=triggered_by, + error=f"基线活动不存在: {baseline_campaign_id}", + ) + return + else: + baseline = resolve_auto_baseline(campaign, session) + if baseline is None: + comparisons.upsert( + campaign_id, + status="failed", + triggered_by=triggered_by, + error="未找到可用的基线活动（计划指纹不一致或历史活动无已完成分析）", + ) + return + + baseline_analysis = analyses.get_by_campaign(baseline.id) + if baseline_analysis is None or baseline_analysis.status != "completed": + comparisons.upsert( + campaign_id, + status="failed", + baseline_campaign_id=baseline.id, + model_config_id=runtime.id, + triggered_by=triggered_by, + error="基线活动没有已完成的智能分析，请先生成基线活动的分析", + ) + return + current_analysis = analyses.get_by_campaign(campaign_id) + if current_analysis is None or current_analysis.status != "completed": + comparisons.upsert( + campaign_id, + status="failed", + baseline_campaign_id=baseline.id, + model_config_id=runtime.id, + triggered_by=triggered_by, + error="本期活动没有已完成的智能分析，请先生成智能分析", + ) + return + + comparisons.upsert( + campaign_id, + status="generating", + baseline_campaign_id=baseline.id, + model_config_id=runtime.id, + triggered_by=triggered_by, + ) + try: + diff = compute_metric_diff( + build_campaign_report_dict(baseline, session), + build_campaign_report_dict(campaign, session), + ) + valid_scenario_ids = {s["scenario_id"] for s in diff["scenarios"]} + result = await narrate_period_comparison( + baseline_analysis=baseline_analysis.get_result(), + current_analysis=current_analysis.get_result(), + metric_diff=diff, + valid_scenario_ids=valid_scenario_ids, + chat_client=chat_client or _gateway_chat_client(runtime), + ) + except Exception as exc: + _logger.warning("活动 %s 周期对比失败: %s", campaign_id, exc) + comparisons.upsert( + campaign_id, + status="failed", + baseline_campaign_id=baseline.id, + model_config_id=runtime.id, + error=str(exc)[:500], + triggered_by=triggered_by, + ) + return + comparisons.upsert( + campaign_id, + status="completed", + baseline_campaign_id=baseline.id, + result=result, + model_config_id=runtime.id, + triggered_by=triggered_by, + ) + finally: + session.close() + + +def start_campaign_comparison( + campaign_id: str, + *, + triggered_by: str, + baseline_campaign_id: Optional[str] = None, +) -> asyncio.Task: + """以后台任务启动对比生成（fire-and-forget；状态经 campaign_period_comparisons 表观测）。""" + return asyncio.create_task( + execute_campaign_comparison( + campaign_id, + triggered_by=triggered_by, + baseline_campaign_id=baseline_campaign_id, + ) + ) diff --git a/backend/agenteval/storage/db.py b/backend/agenteval/storage/db.py index c92eaa5..f635994 100644 --- a/backend/agenteval/storage/db.py +++ b/backend/agenteval/storage/db.py @@ -224,6 +224,33 @@ class CampaignAnalysisDB(SQLModel, table=True): self.result = _json_dumps(result) +class CampaignPeriodComparisonDB(SQLModel, table=True): + """One row per campaign holding its period-comparison (周期对比) state. + + Mirrors ``campaign_analyses`` (one row, upserted per campaign), plus the + baseline campaign this comparison was computed against. + """ + + __tablename__ = "campaign_period_comparisons" + + id: Optional[str] = Field(default_factory=new_uuid, primary_key=True) + campaign_id: str = Field(unique=True, index=True) + baseline_campaign_id: str = Field(index=True) + status: str = "generating" + result: Optional[str] = None + model_config_id: Optional[str] = None + error: Optional[str] = None + triggered_by: str = "manual" + created_at: Optional[datetime] = Field(default_factory=utc_now) + updated_at: Optional[datetime] = Field(default_factory=utc_now) + + def get_result(self) -> Optional[dict[str, Any]]: + return _json_loads(self.result) if self.result else None + + def set_result(self, result: dict[str, Any]) -> None: + self.result = _json_dumps(result) + + class EvalRunDB(SQLModel, table=True): """Database table for evaluation runs.""" diff --git a/backend/agenteval/storage/repository.py b/backend/agenteval/storage/repository.py index 7b3ee3a..4327ab5 100644 --- a/backend/agenteval/storage/repository.py +++ b/backend/agenteval/storage/repository.py @@ -9,6 +9,7 @@ from agenteval.services.model_configs import ModelConfigService from agenteval.storage.db import ( CampaignAnalysisDB, CampaignDB, + CampaignPeriodComparisonDB, EvalResultDB, EvalRunDB, EvalTargetDB, @@ -433,6 +434,52 @@ class CampaignAnalysisRepository: return row +class CampaignPeriodComparisonRepository: + """Repository for period-comparison rows (one per campaign, upserted).""" + + def __init__(self, session: Optional[Session] = None): + self.session = session or get_session() + + def get_by_campaign(self, campaign_id: str) -> Optional[CampaignPeriodComparisonDB]: + statement = select(CampaignPeriodComparisonDB).where( + CampaignPeriodComparisonDB.campaign_id == campaign_id + ) + return self.session.exec(statement).first() + + def upsert( + self, + campaign_id: str, + *, + status: str, + baseline_campaign_id: Optional[str] = None, + result: Optional[dict] = None, + model_config_id: Optional[str] = None, + error: Optional[str] = None, + triggered_by: str = "manual", + ) -> CampaignPeriodComparisonDB: + row = self.get_by_campaign(campaign_id) + if row is None: + row = CampaignPeriodComparisonDB( + campaign_id=campaign_id, + baseline_campaign_id=baseline_campaign_id or "", + ) + row.status = status + if baseline_campaign_id is not None: + row.baseline_campaign_id = baseline_campaign_id + if result is not None: + row.set_result(result) + else: + row.result = None + row.model_config_id = model_config_id + row.error = error + row.triggered_by = triggered_by + row.updated_at = utc_now() + self.session.add(row) + self.session.commit() + self.session.refresh(row) + return row + + class ResultRepository: """Repository for evaluation results.""" diff --git a/backend/agenteval/web/routers/campaigns.py b/backend/agenteval/web/routers/campaigns.py index ebde8dc..4eb2fb9 100644 --- a/backend/agenteval/web/routers/campaigns.py +++ b/backend/agenteval/web/routers/campaigns.py @@ -7,12 +7,19 @@ the live window position and spawned/completed Run counts, and a campaign can be cancelled mid-flight. """ -from fastapi import APIRouter, Depends, HTTPException, Response +from fastapi import APIRouter, Body, Depends, HTTPException, Response from pydantic import BaseModel, Field from sqlmodel import Session from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign +from agenteval.evaluation.comparison import ( + build_campaign_report_dict, + build_comparison_payload, + compute_metric_diff, + resolve_auto_baseline, + start_campaign_comparison, +) from agenteval.evaluation.report import ( build_campaign_timeline, generate_campaign_report, @@ -24,6 +31,7 @@ from agenteval.storage.db import iso_utc, utc_now from agenteval.storage.model_config_repository import ModelConfigRepository from agenteval.storage.repository import ( CampaignAnalysisRepository, + CampaignPeriodComparisonRepository, CampaignRepository, RunRepository, ScenarioRepository, @@ -181,6 +189,101 @@ async def trigger_campaign_analysis(campaign_id: str, session: Session = Depends return {"status": "generating"} +@router.get("/{campaign_id}/comparison") +async def get_campaign_comparison(campaign_id: str, session: Session = Depends(get_db)) -> dict: + """周期对比（v0.8）：自动基线配对 + 叙述行状态 + 机械 diff（现算，不存储）。 + + ``metric_diff`` 对应「生效基线」：已有对比行时为该行记录的基线， + 否则为自动基线；两者都没有时为 null。 + """ + campaign = CampaignRepository(session).get(campaign_id) + if not campaign: + raise HTTPException(status_code=404, detail="campaign not found") + + payload = build_comparison_payload(campaign, session) + row = CampaignPeriodComparisonRepository(session).get_by_campaign(campaign_id) + if row is None: + return {"status": "none", "comparison": None, **payload} + + baseline = CampaignRepository(session).get(row.baseline_campaign_id) + metric_diff = ( + compute_metric_diff( + build_campaign_report_dict(baseline, session), + build_campaign_report_dict(campaign, session), + ) + if baseline is not None + else None + ) + comparison = { + "baseline_campaign_id": row.baseline_campaign_id, + "baseline": ( + { + "id": baseline.id, + "name": baseline.name, + "completed_at": iso_utc(baseline.completed_at), + } + if baseline + else None + ), + "result": row.get_result(), + "error": row.error, + "model_config_id": row.model_config_id, + "triggered_by": row.triggered_by, + "updated_at": iso_utc(row.updated_at), + } + return { + "status": row.status, + "comparison": comparison, + "auto_baseline": payload["auto_baseline"], + "metric_diff": metric_diff, + } + + +class GenerateComparisonRequest(BaseModel): + baseline_campaign_id: str | None = None + + +@router.post("/{campaign_id}/comparison") +async def trigger_campaign_comparison( + campaign_id: str, + request: GenerateComparisonRequest = Body(default=None), + session: Session = Depends(get_db), +) -> dict: + campaign = CampaignRepository(session).get(campaign_id) + if not campaign: + raise HTTPException(status_code=404, detail="campaign not found") + if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING): + raise HTTPException(status_code=400, detail="活动完成后才能生成周期对比") + analyses = CampaignAnalysisRepository(session) + current_analysis = analyses.get_by_campaign(campaign_id) + if current_analysis is None or current_analysis.status != "completed": + raise HTTPException(status_code=400, detail="请先生成本期活动的智能分析") + if resolve_analysis_model(campaign, session) is None: + raise HTTPException( + status_code=400, + detail="未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」，或为该活动指定分析模型", + ) + + explicit_baseline_id = request.baseline_campaign_id if request else None + if explicit_baseline_id: + baseline = CampaignRepository(session).get(explicit_baseline_id) + if baseline is None: + raise HTTPException(status_code=400, detail=f"基线活动不存在: {explicit_baseline_id}") + baseline_analysis = analyses.get_by_campaign(explicit_baseline_id) + if baseline_analysis is None or baseline_analysis.status != "completed": + raise HTTPException(status_code=400, detail="基线活动没有已完成的智能分析，请先生成基线活动的分析") + else: + baseline = resolve_auto_baseline(campaign, session) + if baseline is None: + raise HTTPException( + status_code=400, + detail="未找到自动基线：历史活动中没有同计划指纹且已完成分析的活动，可手动选择基线活动", + ) + + start_campaign_comparison(campaign_id, triggered_by="manual", baseline_campaign_id=baseline.id) + return {"status": "generating"} + + @router.get("/{campaign_id}") async def get_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict: campaign = CampaignRepository(session).get(campaign_id) diff --git a/frontend/web/package-lock.json b/frontend/web/package-lock.json index b833271..3de753b 100644 --- a/frontend/web/package-lock.json +++ b/frontend/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "agenteval-web", - "version": "0.7.0", + "version": "0.8.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agenteval-web", - "version": "0.7.0", + "version": "0.8.0", "dependencies": { "@ant-design/charts": "^2.6.7", "@ant-design/icons": "^6.3.2", diff --git a/frontend/web/package.json b/frontend/web/package.json index 1e3318c..e511c48 100644 --- a/frontend/web/package.json +++ b/frontend/web/package.json @@ -1,6 +1,6 @@ { "name": "agenteval-web", - "version": "0.7.0", + "version": "0.8.0", "private": true, "type": "module", "scripts": { diff --git a/frontend/web/src/api.ts b/frontend/web/src/api.ts index bdc75a0..673c131 100644 --- a/frontend/web/src/api.ts +++ b/frontend/web/src/api.ts @@ -429,6 +429,74 @@ export interface CreateCampaignPayload { analysis_model_config_id?: string | null } +// ── Period comparison (v0.8) ──────────────────────────────────── + +export type ComparisonTrend = 'improving' | 'stable' | 'regressing' + +export interface MetricDeltaPair { + baseline: number | null + current: number | null + delta: number | null +} + +export interface MetricDiffScenario { + scenario_id: string + scenario_name: string + pass_rate: MetricDeltaPair + availability: MetricDeltaPair + avg_latency_ms: MetricDeltaPair +} + +export interface MetricDiff { + overall: { + pass_rate: MetricDeltaPair + availability: MetricDeltaPair + avg_latency_ms: MetricDeltaPair + } + scenarios: MetricDiffScenario[] +} + +export interface ComparisonBaselineInfo { + id: string + name: string + completed_at: string | null +} + +export interface ComparisonProblemEvolution { + status: string + title: string + detail: string + scenario_ids: string[] +} + +export interface ComparisonSuggestionTracking { + text: string + status: string + note: string +} + +export interface ComparisonResult { + trend: ComparisonTrend + summary: string + problem_evolution: ComparisonProblemEvolution[] + suggestion_tracking: ComparisonSuggestionTracking[] +} + +export interface CampaignComparison { + status: CampaignAnalysisStatus + auto_baseline: ComparisonBaselineInfo | null + metric_diff: MetricDiff | null + comparison: { + baseline_campaign_id: string + baseline: ComparisonBaselineInfo | null + result: ComparisonResult | null + error: string | null + model_config_id: string | null + triggered_by: string + updated_at: string | null + } | null +} + export const campaignsApi = { list: () => api.get('/campaigns'), get: (id: string) => api.get(`/campaigns/${id}`), @@ -439,6 +507,12 @@ export const campaignsApi = { api.get<{ entries: CampaignTimelineEntry[] }>(`/campaigns/${id}/timeline`), getAnalysis: (id: string) => api.get(`/campaigns/${id}/analysis`), generateAnalysis: (id: string) => api.post<{ status: string }>(`/campaigns/${id}/analysis`), + getComparison: (id: string) => api.get(`/campaigns/${id}/comparison`), + generateComparison: (id: string, baselineCampaignId?: string) => + api.post<{ status: string }>( + `/campaigns/${id}/comparison`, + baselineCampaignId ? { baseline_campaign_id: baselineCampaignId } : {}, + ), downloadReport: async (id: string) => { const res = await api.get(`/campaigns/${id}/report/markdown`, { responseType: 'blob' }) const url = URL.createObjectURL(res.data as Blob) diff --git a/frontend/web/src/pages/Campaigns.tsx b/frontend/web/src/pages/Campaigns.tsx index c16874f..1e6d014 100644 --- a/frontend/web/src/pages/Campaigns.tsx +++ b/frontend/web/src/pages/Campaigns.tsx @@ -8,7 +8,7 @@ import { PlusOutlined, ReloadOutlined, StopOutlined, BarChartOutlined, FileMarkdownOutlined, MinusCircleOutlined, QuestionCircleOutlined, RocketOutlined, CheckCircleOutlined, SafetyOutlined, ClockCircleOutlined, - RobotOutlined, + RobotOutlined, SwapOutlined, } from '@ant-design/icons' import { Line, Bar } from '@ant-design/charts' import PageWrapper from '../components/PageWrapper' @@ -17,6 +17,7 @@ import { campaignsApi, targetsApi, scenariosApi, runsApi, modelConfigsApi, type CampaignListItem, type CampaignReport, type Target, type Scenario, type Run, type CampaignTimelineEntry, type ModelConfig, type CampaignAnalysis, + type CampaignComparison, type MetricDiff, } from '../api' import { passRateColor } from '../utils/colors' import { shortDateTime, toDate } from '../utils/date' @@ -42,6 +43,25 @@ const SEVERITY_META: Record = { low: { label: '低', color: 'blue' }, } +const TREND_META: Record = { + improving: { label: '改善', color: 'green' }, + stable: { label: '平稳', color: 'blue' }, + regressing: { label: '退化', color: 'red' }, +} + +const EVOLUTION_META: Record = { + new: { label: '新增', color: 'red' }, + persisting: { label: '持续', color: 'orange' }, + resolved: { label: '消解', color: 'green' }, +} + +const TRACKING_META: Record = { + addressed: { label: '已落实', color: 'green' }, + partial: { label: '部分落实', color: 'orange' }, + unaddressed: { label: '未落实', color: 'default' }, + new: { label: '新增', color: 'blue' }, +} + const WINDOW_OPTIONS = [6, 12, 24, 48, 72].map((h) => ({ label: `${h} 小时`, value: h * 3600 })) const UNIT_OPTIONS = [ @@ -125,6 +145,9 @@ export default function CampaignsPage() { const [reportTimeline, setReportTimeline] = useState([]) const [analysis, setAnalysis] = useState(null) const [analysisBusy, setAnalysisBusy] = useState(false) + const [comparison, setComparison] = useState(null) + const [comparisonBusy, setComparisonBusy] = useState(false) + const [baselineChoice, setBaselineChoice] = useState(undefined) const [expandedIds, setExpandedIds] = useState([]) const [timelines, setTimelines] = useState>({}) @@ -233,18 +256,22 @@ export default function CampaignsPage() { setReportRuns([]) setReportTimeline([]) setAnalysis(null) + setComparison(null) + setBaselineChoice(undefined) } try { - const [rep, runs, tl, ana] = await Promise.all([ + const [rep, runs, tl, ana, cmp] = await Promise.all([ campaignsApi.report(campaignId), runsApi.list(), campaignsApi.timeline(campaignId), campaignsApi.getAnalysis(campaignId), + campaignsApi.getComparison(campaignId), ]) setReport(rep.data) setReportRuns(runs.data.filter((r) => r.campaign_id === campaignId)) setReportTimeline(tl.data.entries) setAnalysis(ana.data) + setComparison(cmp.data) } finally { if (!silent) setReportLoading(false) } @@ -288,6 +315,28 @@ export default function CampaignsPage() { activeKey === '/campaigns' && reportOpen && !!reportId && analysis?.status === 'generating', ) + const generateComparison = async (baselineCampaignId?: string) => { + if (!reportId) return + setComparisonBusy(true) + try { + await campaignsApi.generateComparison(reportId, baselineCampaignId) + const res = await campaignsApi.getComparison(reportId) + setComparison(res.data) + } finally { + setComparisonBusy(false) + } + } + + // 对比生成中同样轮询，直到 completed/failed。 + usePolling( + () => { + if (!reportId) return + void campaignsApi.getComparison(reportId).then((res) => setComparison(res.data)) + }, + POLL_INTERVAL_MS, + activeKey === '/campaigns' && reportOpen && !!reportId && comparison?.status === 'generating', + ) + // Grow the expanded timeline of any still-running campaign as new child Runs // spawn. Completed/cancelled campaigns are fetched once on expand. const activeExpandedIds = expandedIds.filter( @@ -636,6 +685,231 @@ export default function CampaignsPage() { ) } + // ── 周期对比区块 ───────────────────────────────────────────────────── + const comparisonReady = analysis?.status === 'completed' + const comparisonModelName = comparison?.comparison?.model_config_id + ? (data?.modelConfigs ?? []).find((m) => m.id === comparison.comparison?.model_config_id)?.name + ?? comparison.comparison.model_config_id.slice(0, 8) + : null + + const baselineCandidates = report + ? campaigns + .filter((c) => c.target_id === report.target_id + && c.id !== report.campaign_id + && !isActiveStatus(c.status)) + .slice() + .sort((a, b) => (b.completed_at ?? '').localeCompare(a.completed_at ?? '')) + : [] + + const comparisonButton = (label: string, baselineCampaignId?: string, requireBaseline = false) => ( + + + + ) + + const fmtMetricValue = (metric: 'pass_rate' | 'availability' | 'avg_latency_ms', v: number | null): string => { + if (v == null) return '—' + return metric === 'avg_latency_ms' ? `${Math.round(v)}ms` : fmtPct(v) + } + + const fmtMetricDelta = (metric: 'pass_rate' | 'availability' | 'avg_latency_ms', d: number | null): string => { + if (d == null) return '—' + const sign = d > 0 ? '+' : '' + return metric === 'avg_latency_ms' ? `${sign}${Math.round(d * 10) / 10}ms` : `${sign}${(d * 100).toFixed(1)}pp` + } + + const deltaColor = (metric: 'pass_rate' | 'availability' | 'avg_latency_ms', d: number | null): string => { + if (d == null || d === 0) return colors.textMuted + // 时延下降是好事；比率上升是好事。 + const good = metric === 'avg_latency_ms' ? d < 0 : d > 0 + return good ? '#52c41a' : '#ff4d4f' + } + + const metricDiffTable = (diff: MetricDiff) => { + const rows = [ + { key: 'overall', name: '整窗（总体）', ...diff.overall }, + ...diff.scenarios.map((s) => ({ + key: s.scenario_id, + name: s.scenario_name || scenarioNames[s.scenario_id] || s.scenario_id.slice(0, 8), + pass_rate: s.pass_rate, availability: s.availability, avg_latency_ms: s.avg_latency_ms, + })), + ] + const metricCols = (['pass_rate', 'availability', 'avg_latency_ms'] as const).map((metric) => ({ + title: metric === 'pass_rate' ? '通过率' : metric === 'availability' ? '可用性' : '平均时延', + key: metric, + render: (_: unknown, r: typeof rows[number]) => { + const pair = r[metric] + return ( + + {fmtMetricValue(metric, pair.baseline)} + → + {fmtMetricValue(metric, pair.current)} + + {fmtMetricDelta(metric, pair.delta)} + + + ) + }, + })) + return ( + + ) + } + + const renderComparisonSection = () => { + const status = comparison?.status ?? 'none' + if (status === 'generating') { + return ( +
+ + + 正在生成周期对比叙述，通常需要几十秒… + +
+ ) + } + if (status === 'failed') { + return ( +
+ +
+ ) + } + const entry = comparison?.comparison ?? null + if (!entry || !entry.result) { + const autoBaseline = comparison?.auto_baseline ?? null + return ( +
+
+ {autoBaseline + ? <>已找到自动基线「{autoBaseline.name}」（{shortDateTime(autoBaseline.completed_at)}），可生成对比叙述。 + : '未找到自动基线（需同计划、正式线、更早完成且已有智能分析的历史活动）。可手动选择基线。'} +
+ + {autoBaseline + ? comparisonButton('生成对比') + : ( + <> +