From dd3b9a5e91a966db454840435ea0ad7d1d657ef6 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Mon, 3 Aug 2026 14:26:20 +0800 Subject: [PATCH] =?UTF-8?q?refactor(comparison):=20=E8=AF=84=E5=AE=A1?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20=E2=80=94=20=E5=85=B1=E4=BA=AB=20gateway?= =?UTF-8?q?=5Fchat=5Fclient=E3=80=81=E6=8C=87=E6=A0=87=E5=85=83=E8=A1=A8?= =?UTF-8?q?=E3=80=81=E5=AF=B9=E6=AF=94=E5=8C=BA=E5=9D=97=E7=BB=84=E4=BB=B6?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - analysis/comparison 重复的 _gateway_chat_client 提取为共享 gateway_chat_client - Campaigns.tsx 周期对比区块抽为 PeriodComparisonSection 组件，指标格式化 收敛为单一 METRICS 元表（消除三处 metric 分支级联） - 基线下拉排除无 completed_at 的终态活动（选中必 400） - spec/issue 03 追认 delta 按好坏着色口径与 GET 生效基线合并口径 --- .../03-report-drawer-comparison-section.md | 4 +- .scratch/v0.8/spec.md | 4 +- backend/agenteval/evaluation/analysis.py | 5 +- backend/agenteval/evaluation/comparison.py | 15 +- .../components/PeriodComparisonSection.tsx | 286 ++++++++++++++++++ frontend/web/src/pages/Campaigns.tsx | 274 ++--------------- 6 files changed, 312 insertions(+), 276 deletions(-) create mode 100644 frontend/web/src/components/PeriodComparisonSection.tsx diff --git a/.scratch/v0.8/issues/03-report-drawer-comparison-section.md b/.scratch/v0.8/issues/03-report-drawer-comparison-section.md index be0b332..f3e8c7f 100644 --- a/.scratch/v0.8/issues/03-report-drawer-comparison-section.md +++ b/.scratch/v0.8/issues/03-report-drawer-comparison-section.md @@ -1,6 +1,6 @@ # 03 — 报告抽屉「周期对比」区块 -**What to build:** 报告抽屉在「智能分析」之下新增「周期对比」区块。状态行：生成中（spinner + 5s 轮询）、失败（错误原因 + 重试）、未生成（「生成对比」按钮；无自动基线时显示基线选择器——同对象历史终态活动下拉，选定后生成）。已生成依次渲染：趋势徽章（improving 绿 / stable 蓝 / regressing 红）+ 总结 → 指标变化表（整窗行 + 分场景行，delta 正绿负红、None 显示 —）→ 问题演化列表（新增红 / 持续橙 / 消解绿 Tag + 标题 + 说明 + 场景 Tag）→ 建议追踪（已落实绿 / 部分橙 / 未落实灰 / 新增蓝）。展示基线活动名与完成时间、所用分析模型名与生成时间。生成按钮仅终态活动可用。 +**What to build:** 报告抽屉在「智能分析」之下新增「周期对比」区块。状态行：生成中（spinner + 5s 轮询）、失败（错误原因 + 重试）、未生成（「生成对比」按钮；无自动基线时显示基线选择器——同对象历史终态活动下拉，选定后生成）。已生成依次渲染：趋势徽章（improving 绿 / stable 蓝 / regressing 红）+ 总结 → 指标变化表（整窗行 + 分场景行，delta 按好坏着色——比率升为绿、时延降为绿，反之红、None 显示 —）→ 问题演化列表（新增红 / 持续橙 / 消解绿 Tag + 标题 + 说明 + 场景 Tag）→ 建议追踪（已落实绿 / 部分橙 / 未落实灰 / 新增蓝）。展示基线活动名与完成时间、所用分析模型名与生成时间。生成按钮仅终态活动可用。 **Blocked by:** 02（API 与结果结构） @@ -8,7 +8,7 @@ - [x] `api.ts`：对比类型 + `getComparison` / `generateComparison(id, baselineId?)` - [x] 区块按状态渲染（生成中轮询复用 `usePolling`） -- [x] 已生成渲染：趋势徽章 + 总结 / 指标 delta 表（正绿负红）/ 问题演化 Tag / 建议追踪状态 +- [x] 已生成渲染：趋势徽章 + 总结 / 指标 delta 表（按好坏着色）/ 问题演化 Tag / 建议追踪状态 - [x] 无自动基线时的基线选择器（同对象终态活动下拉） - [x] 基线活动名 + 完成时间 + 模型名 + 生成时间展示；按钮终态门控 - [x] `npx tsc --noEmit` + `npm run build` 通过；浏览器实操验证 diff --git a/.scratch/v0.8/spec.md b/.scratch/v0.8/spec.md index 55ca81c..8cdad6a 100644 --- a/.scratch/v0.8/spec.md +++ b/.scratch/v0.8/spec.md @@ -57,14 +57,14 @@ ### API -- `GET /api/campaigns/{id}/comparison`：无行 → 空态 `{"status": "none"}`；有行 → 机械 diff（现算）+ 叙述行合并返回；附 `baseline_campaign_id` 与自动配对结果，前端据此渲染或引导选基线 +- `GET /api/campaigns/{id}/comparison`：无行 → 空态 `{"status": "none"}`；有行 → 机械 diff（现算）+ 叙述行合并返回；附 `baseline_campaign_id` 与自动配对结果，前端据此渲染或引导选基线。`metric_diff` 跟随生效基线：有对比行按行内基线算，否则按自动基线算 - `POST /api/campaigns/{id}/comparison`：body 可选 `{"baseline_campaign_id": "..."}`；非终态活动 400；当前活动无 completed 分析 400（先生成分析）；无模型 400 引导；成功 → 后台任务 + `{"status": "generating"}` - 两期都有 completed 分析是生成前提（基线无分析 → 400 提示先生成基线分析） ### 前端（报告抽屉「周期对比」区块） - 位置：「智能分析」区块之下，复用区块状态行范式（生成中 5s 轮询 / 失败重试 / 未生成按钮） -- 已生成：趋势徽章 + 总结 → 指标变化表（整窗 + 分场景 delta，正绿负红）→ 问题演化（新增红/持续橙/消解绿 Tag）→ 建议追踪（已落实绿/部分橙/未落实灰/新增蓝） +- 已生成：趋势徽章 + 总结 → 指标变化表（整窗 + 分场景 delta，按好坏着色——比率升为绿、时延降为绿，反之红）→ 问题演化（新增红/持续橙/消解绿 Tag）→ 建议追踪（已落实绿/部分橙/未落实灰/新增蓝） - 无自动基线：基线选择器（同对象历史终态活动下拉）+ 生成按钮 - 展示基线活动名与完成时间、所用分析模型名与生成时间 diff --git a/backend/agenteval/evaluation/analysis.py b/backend/agenteval/evaluation/analysis.py index 073135f..a8cf63d 100644 --- a/backend/agenteval/evaluation/analysis.py +++ b/backend/agenteval/evaluation/analysis.py @@ -262,7 +262,8 @@ def _maybe_enqueue_period_comparison(campaign: Campaign, session: Session) -> No _logger.warning("活动 %s 自动周期对比跳过: %s", campaign.id, exc) -def _gateway_chat_client(runtime: ModelRuntimeConfig) -> ChatClient: +def gateway_chat_client(runtime: ModelRuntimeConfig) -> ChatClient: + """Shared ChatClient factory for analysis/comparison background executors.""" gateway = ModelGateway(timeout=180.0) async def _chat(messages: list[dict[str, str]]) -> str: @@ -299,7 +300,7 @@ async def execute_campaign_analysis( model_config_id=runtime.id, triggered_by=triggered_by, ) try: - client = chat_client or _gateway_chat_client(runtime) + client = chat_client or gateway_chat_client(runtime) runs = RunRepository(session).list_by_campaign(campaign_id) scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()} report = generate_campaign_report(campaign, runs, scenario_names=scenario_names) diff --git a/backend/agenteval/evaluation/comparison.py b/backend/agenteval/evaluation/comparison.py index c94b933..1243241 100644 --- a/backend/agenteval/evaluation/comparison.py +++ b/backend/agenteval/evaluation/comparison.py @@ -14,11 +14,9 @@ from typing import Any, Optional from sqlmodel import Session -from agenteval.evaluation.analysis import ChatClient, resolve_analysis_model +from agenteval.evaluation.analysis import ChatClient, gateway_chat_client, resolve_analysis_model from agenteval.evaluation.report import generate_campaign_report -from agenteval.model_gateway import ModelGateway from agenteval.models import Campaign, EvalRun -from agenteval.services.model_configs import ModelRuntimeConfig from agenteval.storage.db import get_session, iso_utc, utc_now from agenteval.storage.repository import ( CampaignAnalysisRepository, @@ -277,15 +275,6 @@ async def narrate_period_comparison( } -def _gateway_chat_client(runtime: ModelRuntimeConfig) -> ChatClient: - gateway = ModelGateway(timeout=180.0) - - async def _chat(messages: list[dict[str, str]]) -> str: - return await gateway.chat(runtime, messages, temperature=0.2) - - return _chat - - async def execute_campaign_comparison( campaign_id: str, *, @@ -378,7 +367,7 @@ async def execute_campaign_comparison( current_analysis=current_analysis.get_result(), metric_diff=diff, valid_scenario_ids=valid_scenario_ids, - chat_client=chat_client or _gateway_chat_client(runtime), + chat_client=chat_client or gateway_chat_client(runtime), ) except Exception as exc: _logger.warning("活动 %s 周期对比失败: %s", campaign_id, exc) diff --git a/frontend/web/src/components/PeriodComparisonSection.tsx b/frontend/web/src/components/PeriodComparisonSection.tsx new file mode 100644 index 0000000..a97a624 --- /dev/null +++ b/frontend/web/src/components/PeriodComparisonSection.tsx @@ -0,0 +1,286 @@ +import { useState, type CSSProperties } from 'react' +import { Alert, Button, Select, Space, Spin, Table, Tag, Tooltip } from 'antd' +import { + type CampaignComparison, type CampaignListItem, type MetricDeltaPair, + type MetricDiff, type ModelConfig, +} from '../api' +import { shortDateTime } from '../utils/date' +import { colors } from '../tokens' + +const TREND_META: Record = { + improving: { label: '改善', color: 'green' }, + stable: { label: '平稳', color: 'blue' }, + regressing: { label: '退化', color: 'red' }, +} + +const EVOLUTION_META: Record = { + new: { label: '新增', color: 'red' }, + persisting: { label: '持续', color: 'orange' }, + resolved: { label: '消解', color: 'green' }, +} + +const TRACKING_META: Record = { + addressed: { label: '已落实', color: 'green' }, + partial: { label: '部分落实', color: 'orange' }, + unaddressed: { label: '未落实', color: 'default' }, + new: { label: '新增', color: 'blue' }, +} + +type MetricKey = 'pass_rate' | 'availability' | 'avg_latency_ms' + +const fmtPct = (v: number) => `${(v * 100).toFixed(1)}%` +const signed = (d: number, body: string) => `${d > 0 ? '+' : ''}${body}` + +/** Single metric meta table: label, formatting and which direction is good. */ +const METRICS: { + key: MetricKey + label: string + goodDirection: 'up' | 'down' + fmt: (v: number) => string + fmtDelta: (d: number) => string +}[] = [ + { key: 'pass_rate', label: '通过率', goodDirection: 'up', fmt: fmtPct, fmtDelta: (d) => signed(d, `${(d * 100).toFixed(1)}pp`) }, + { key: 'availability', label: '可用性', goodDirection: 'up', fmt: fmtPct, fmtDelta: (d) => signed(d, `${(d * 100).toFixed(1)}pp`) }, + { key: 'avg_latency_ms', label: '平均时延', goodDirection: 'down', fmt: (v) => `${Math.round(v)}ms`, fmtDelta: (d) => signed(d, `${Math.round(d * 10) / 10}ms`) }, +] + +const boxStyle: CSSProperties = { + border: `1px solid ${colors.border}`, borderRadius: 8, + padding: '12px 14px', marginBottom: 16, +} + +interface Props { + comparison: CampaignComparison | null + /** 活动已终态（按钮门控之一） */ + terminal: boolean + /** 当前活动智能分析已 completed（对比前提） */ + analysisCompleted: boolean + targetId: string + currentCampaignId: string + campaigns: CampaignListItem[] + scenarioNames: Record + modelConfigs: ModelConfig[] + onGenerate: (baselineCampaignId?: string) => Promise +} + +export default function PeriodComparisonSection({ + comparison, terminal, analysisCompleted, targetId, currentCampaignId, + campaigns, scenarioNames, modelConfigs, onGenerate, +}: Props) { + const [busy, setBusy] = useState(false) + const [baselineChoice, setBaselineChoice] = useState(undefined) + + const generate = async (baselineCampaignId?: string) => { + setBusy(true) + try { + await onGenerate(baselineCampaignId) + } finally { + setBusy(false) + } + } + + const baselineCandidates = campaigns + .filter((c) => c.target_id === targetId + && c.id !== currentCampaignId + && c.completed_at != null + && !(c.status === 'planned' || c.status === 'running')) + .slice() + .sort((a, b) => (b.completed_at ?? '').localeCompare(a.completed_at ?? '')) + + const button = (label: string, baselineCampaignId?: string, requireBaseline = false) => ( + + + + ) + + const metricDiffTable = (diff: MetricDiff) => { + const rows = [ + { key: 'overall', name: '整窗（总体）', ...diff.overall }, + ...diff.scenarios.map((s) => ({ + key: s.scenario_id, + name: s.scenario_name || scenarioNames[s.scenario_id] || s.scenario_id.slice(0, 8), + pass_rate: s.pass_rate, availability: s.availability, avg_latency_ms: s.avg_latency_ms, + })), + ] + const renderPair = (metric: typeof METRICS[number]) => (_: unknown, r: typeof rows[number]) => { + const pair: MetricDeltaPair = r[metric.key] + const good = metric.goodDirection === 'down' ? (pair.delta ?? 0) < 0 : (pair.delta ?? 0) > 0 + const deltaColor = pair.delta == null || pair.delta === 0 + ? colors.textMuted + : good ? '#52c41a' : '#ff4d4f' + return ( + + + {pair.baseline == null ? '—' : metric.fmt(pair.baseline)} + + → + {pair.current == null ? '—' : metric.fmt(pair.current)} + + {pair.delta == null ? '—' : metric.fmtDelta(pair.delta)} + + + ) + } + return ( + ({ title: m.label, key: m.key, render: renderPair(m) })), + ]} + style={{ marginBottom: 12 }} + /> + ) + } + + const status = comparison?.status ?? 'none' + if (status === 'generating') { + return ( +
+ + + 正在生成周期对比叙述，通常需要几十秒… + +
+ ) + } + if (status === 'failed') { + return ( +
+ +
+ ) + } + const entry = comparison?.comparison ?? null + if (!entry || !entry.result) { + const autoBaseline = comparison?.auto_baseline ?? null + return ( +
+
+ {autoBaseline + ? <>已找到自动基线「{autoBaseline.name}」（{shortDateTime(autoBaseline.completed_at)}），可生成对比叙述。 + : '未找到自动基线（需同计划、正式线、更早完成且已有智能分析的历史活动）。可手动选择基线。'} +
+ + {autoBaseline + ? button('生成对比') + : ( + <> +
- ) - } - - const renderComparisonSection = () => { - const status = comparison?.status ?? 'none' - if (status === 'generating') { - return ( -
- - - 正在生成周期对比叙述，通常需要几十秒… - -
- ) - } - if (status === 'failed') { - return ( -
- -
- ) - } - const entry = comparison?.comparison ?? null - if (!entry || !entry.result) { - const autoBaseline = comparison?.auto_baseline ?? null - return ( -
-
- {autoBaseline - ? <>已找到自动基线「{autoBaseline.name}」（{shortDateTime(autoBaseline.completed_at)}），可生成对比叙述。 - : '未找到自动基线（需同计划、正式线、更早完成且已有智能分析的历史活动）。可手动选择基线。'} -
- - {autoBaseline - ? comparisonButton('生成对比') - : ( - <> -