diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index 1abb7ca..cac96cb 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -323,6 +323,22 @@ def _aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]: } +def summarize_campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict[str, Any]: + """Compact list-row progress: completed vs *planned* total, plus pass_rate. + + Unlike ``campaign_progress`` (live window position), this powers the list + view. ``planned_total`` is the sum of plan-entry counts — a fixed target the + campaign works toward, so the progress bar fills from 0 rather than tracking + a growing spawned count. ``overall_pass_rate`` reuses ``_aggregate_runs`` so + the list figure matches the report exactly (ADR-0002: failures count as 0.0). + """ + return { + "completed_runs": sum(1 for r in runs if r.status == RunStatus.COMPLETED), + "planned_total": sum(entry.count for entry in campaign.plan), + "overall_pass_rate": _aggregate_runs(runs)["pass_rate"], + } + + def generate_campaign_report( campaign: Campaign, runs: list[EvalRun], diff --git a/backend/agenteval/web/routers/campaigns.py b/backend/agenteval/web/routers/campaigns.py index fc034d8..ac18a51 100644 --- a/backend/agenteval/web/routers/campaigns.py +++ b/backend/agenteval/web/routers/campaigns.py @@ -12,7 +12,11 @@ from pydantic import BaseModel, Field from sqlmodel import Session from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign -from agenteval.evaluation.report import generate_campaign_report, render_campaign_markdown_report +from agenteval.evaluation.report import ( + generate_campaign_report, + render_campaign_markdown_report, + summarize_campaign_progress, +) from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus from agenteval.storage.db import utc_now from agenteval.storage.repository import ( @@ -36,7 +40,15 @@ class CreateCampaignRequest(BaseModel): @router.get("") async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]: - return [c.model_dump() for c in CampaignRepository(session).list_all()] + repo = CampaignRepository(session) + run_repo = RunRepository(session) + result = [] + for campaign in repo.list_all(): + data = campaign.model_dump() + runs = run_repo.list_by_campaign(campaign.id) + data["progress"] = summarize_campaign_progress(campaign, runs) + result.append(data) + return result @router.post("") diff --git a/frontend/web/src/api.ts b/frontend/web/src/api.ts index a0da90a..5ed1e46 100644 --- a/frontend/web/src/api.ts +++ b/frontend/web/src/api.ts @@ -135,7 +135,7 @@ export interface ModelConfigReference { purpose: string } -export type RunTrigger = 'manual' | 'ai_assistant' | 'cli' +export type RunTrigger = 'manual' | 'ai_assistant' | 'cli' | 'campaign' export interface Run { id: string @@ -298,6 +298,12 @@ export interface CampaignProgress { completed_runs: number } +export interface CampaignListProgress { + completed_runs: number + planned_total: number + overall_pass_rate: number | null +} + export interface Campaign { id: string name: string @@ -312,6 +318,10 @@ export interface Campaign { progress?: CampaignProgress } +export interface CampaignListItem extends Omit { + progress: CampaignListProgress +} + export interface CampaignTrendBucket { bucket_index: number start_seconds: number @@ -360,7 +370,7 @@ export interface CreateCampaignPayload { } export const campaignsApi = { - list: () => api.get('/campaigns'), + list: () => api.get('/campaigns'), get: (id: string) => api.get(`/campaigns/${id}`), create: (data: CreateCampaignPayload) => api.post('/campaigns', data), cancel: (id: string) => api.post(`/campaigns/${id}/cancel`), diff --git a/frontend/web/src/pages/Campaigns.tsx b/frontend/web/src/pages/Campaigns.tsx index 7841242..b6cfa11 100644 --- a/frontend/web/src/pages/Campaigns.tsx +++ b/frontend/web/src/pages/Campaigns.tsx @@ -12,12 +12,13 @@ import { Line } from '@ant-design/charts' import PageWrapper from '../components/PageWrapper' import { campaignsApi, targetsApi, scenariosApi, runsApi, - type Campaign, type CampaignReport, type Target, type Scenario, type Run, + type CampaignListItem, type CampaignReport, type Target, type Scenario, type Run, } from '../api' import { passRateColor } from '../utils/colors' import { shortDateTime } from '../utils/date' import { useOnTabActive } from '../hooks/useOnTabActive' -import { colors, statusColors, statusLabels } from '../tokens' +import { useTabStore } from '../stores/tabStore' +import { colors, statusColors, statusLabels, triggerColors, triggerLabels } from '../tokens' const CAMPAIGN_STATUS: Record = { planned: { label: '计划中', color: 'default' }, @@ -29,6 +30,10 @@ const CAMPAIGN_STATUS: Record = { const WINDOW_OPTIONS = [6, 12, 24, 48, 72].map((h) => ({ label: `${h} 小时`, value: h * 3600 })) +const POLL_INTERVAL_MS = 5000 + +const isActiveStatus = (status: string) => status === 'planned' || status === 'running' + function fmtWindow(seconds: number): string { if (seconds % 3600 === 0) return `${seconds / 3600}h` if (seconds % 60 === 0) return `${seconds / 60}m` @@ -47,10 +52,10 @@ interface PlanFormEntry { export default function CampaignsPage() { const navigate = useNavigate() - const [campaigns, setCampaigns] = useState([]) + const activeKey = useTabStore((s) => s.activeKey) + const [campaigns, setCampaigns] = useState([]) const [targets, setTargets] = useState([]) const [scenarios, setScenarios] = useState([]) - const [reportMap, setReportMap] = useState>({}) const [loading, setLoading] = useState(false) const [createOpen, setCreateOpen] = useState(false) @@ -64,8 +69,8 @@ export default function CampaignsPage() { const targetName = (id: string) => targets.find((t) => t.id === id)?.name ?? id.slice(0, 8) - const loadData = async () => { - setLoading(true) + const loadData = async (silent = false) => { + if (!silent) setLoading(true) try { const [c, t, s] = await Promise.all([ campaignsApi.list(), targetsApi.list(), scenariosApi.list(), @@ -73,24 +78,23 @@ export default function CampaignsPage() { setCampaigns(c.data) setTargets(t.data) setScenarios(s.data) - // Fetch each campaign's aggregate report so the list can show progress - // and overall pass_rate at a glance (activity counts are small). - const reports = await Promise.all( - c.data.map((camp) => - campaignsApi.report(camp.id).then((r) => r.data).catch(() => null), - ), - ) - const map: Record = {} - reports.forEach((r) => { if (r) map[r.campaign_id] = r }) - setReportMap(map) } finally { - setLoading(false) + if (!silent) setLoading(false) } } useEffect(() => { loadData() }, []) useOnTabActive('/campaigns', loadData) + // Poll the list while this tab is active and a campaign is still working — + // compressed dev-line campaigns change fast. Stop once all are terminal. + const hasActiveCampaign = campaigns.some((c) => isActiveStatus(c.status)) + useEffect(() => { + if (activeKey !== '/campaigns' || !hasActiveCampaign) return + const id = setInterval(() => loadData(true), POLL_INTERVAL_MS) + return () => clearInterval(id) + }, [activeKey, hasActiveCampaign]) + const openCreate = () => { form.setFieldsValue({ name: '', target_id: undefined, window_seconds: 24 * 3600, time_scale: 1, @@ -128,50 +132,63 @@ export default function CampaignsPage() { loadData() } - const openReport = async (campaign: Campaign) => { - setReportOpen(true) - setReportLoading(true) - setReport(null) - setReportRuns([]) + const fetchReport = async (campaignId: string, silent = false) => { + if (!silent) { + setReportLoading(true) + setReport(null) + setReportRuns([]) + } try { const [rep, runs] = await Promise.all([ - campaignsApi.report(campaign.id), + campaignsApi.report(campaignId), runsApi.list(), ]) setReport(rep.data) - setReportRuns(runs.data.filter((r) => r.campaign_id === campaign.id)) + setReportRuns(runs.data.filter((r) => r.campaign_id === campaignId)) } finally { - setReportLoading(false) + if (!silent) setReportLoading(false) } } + const openReport = (campaign: CampaignListItem) => { + setReportOpen(true) + fetchReport(campaign.id) + } + + // Keep the open report drawer live while its campaign is still running. + const reportId = report?.campaign_id ?? null + const reportCampaignActive = campaigns.some( + (c) => c.id === reportId && isActiveStatus(c.status), + ) + useEffect(() => { + if (activeKey !== '/campaigns' || !reportOpen || !reportId || !reportCampaignActive) return + const id = setInterval(() => fetchReport(reportId, true), POLL_INTERVAL_MS) + return () => clearInterval(id) + }, [activeKey, reportOpen, reportId, reportCampaignActive]) + const columns = [ { title: '名称', dataIndex: 'name', key: 'name' }, - { title: '评测对象', key: 'target', render: (_: unknown, c: Campaign) => targetName(c.target_id) }, - { title: '窗口', key: 'window', render: (_: unknown, c: Campaign) => fmtWindow(c.window_seconds) }, + { title: '评测对象', key: 'target', render: (_: unknown, c: CampaignListItem) => targetName(c.target_id) }, + { title: '窗口', key: 'window', render: (_: unknown, c: CampaignListItem) => fmtWindow(c.window_seconds) }, { title: '倍速', key: 'scale', - render: (_: unknown, c: Campaign) => (c.time_scale === 1 ? '实时' : `×${c.time_scale}`), + render: (_: unknown, c: CampaignListItem) => (c.time_scale === 1 ? '实时' : `×${c.time_scale}`), }, { title: '状态', key: 'status', - render: (_: unknown, c: Campaign) => { + render: (_: unknown, c: CampaignListItem) => { const meta = CAMPAIGN_STATUS[c.status] ?? { label: c.status, color: 'default' } return {meta.label} }, }, { title: '进度', key: 'progress', - render: (_: unknown, c: Campaign) => { - const rep = reportMap[c.id] - if (!rep) return '—' - return `${rep.summary.completed_runs}/${rep.summary.total_runs}` - }, + render: (_: unknown, c: CampaignListItem) => `${c.progress.completed_runs}/${c.progress.planned_total}`, }, { title: '整窗通过率', key: 'pass_rate', - render: (_: unknown, c: Campaign) => { - const rate = reportMap[c.id]?.summary.overall_pass_rate + render: (_: unknown, c: CampaignListItem) => { + const rate = c.progress.overall_pass_rate if (rate == null) return '—' return ( ( + render: (_: unknown, c: CampaignListItem) => ( - {(c.status === 'planned' || c.status === 'running') && ( + {isActiveStatus(c.status) && ( cancelCampaign(c.id)}> @@ -243,7 +260,19 @@ export default function CampaignsPage() { { title: '子运行', dataIndex: 'id', key: 'id', render: (id: string) => {id.slice(0, 8)} }, { title: '场景', key: 'scenario', - render: (_: unknown, r: Run) => scenarios.find((s) => s.id === r.scenario_id)?.name ?? r.scenario_id.slice(0, 8), + render: (_: unknown, r: Run) => ( + + {scenarios.find((s) => s.id === r.scenario_id)?.name ?? r.scenario_id.slice(0, 8)} + {r.scenario_version != null && v{r.scenario_version}} + + ), + }, + { + title: '来源', key: 'trigger', + render: (_: unknown, r: Run) => { + const t = r.triggered_by ?? '' + return {triggerLabels[t] ?? t} + }, }, { title: '状态', key: 'status', @@ -277,7 +306,7 @@ export default function CampaignsPage() { fullHeight extra={ - } diff --git a/frontend/web/src/tokens.ts b/frontend/web/src/tokens.ts index e89fcfe..ea50465 100644 --- a/frontend/web/src/tokens.ts +++ b/frontend/web/src/tokens.ts @@ -47,12 +47,14 @@ export const triggerLabels: Record = { manual: '手动', ai_assistant: 'AI 助手', cli: 'CLI', + campaign: '活动', } export const triggerColors: Record = { manual: 'default', ai_assistant: 'purple', cli: 'blue', + campaign: 'cyan', } export const spacing = { diff --git a/tests/integration/test_campaigns_api.py b/tests/integration/test_campaigns_api.py index f140261..ac9d134 100644 --- a/tests/integration/test_campaigns_api.py +++ b/tests/integration/test_campaigns_api.py @@ -132,6 +132,31 @@ async def test_list_campaigns_after_create(client, seeded_db): assert listing[0]["name"] == "24h-cycle" +async def test_list_campaigns_embeds_progress(client, seeded_db): + from agenteval.models import EvalRun, RunStatus + + # _valid_payload plan totals 2 + 1 = 3 planned runs. + campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] + repo = RunRepository(seeded_db) + repo.create(EvalRun( + target_id="t-1", scenario_id="s-1", campaign_id=campaign_id, + status=RunStatus.COMPLETED, + summary={"pass_rate": 1.0, "avg_latency_ms": 100}, + )) + repo.create(EvalRun( + target_id="t-1", scenario_id="s-1", campaign_id=campaign_id, + status=RunStatus.COMPLETED, + summary={"pass_rate": 0.0, "avg_latency_ms": 200}, + )) + + listing = (await client.get("/api/campaigns")).json() + assert len(listing) == 1 + progress = listing[0]["progress"] + assert progress["planned_total"] == 3 # Σ plan.count + assert progress["completed_runs"] == 2 + assert progress["overall_pass_rate"] == 0.5 + + async def test_get_campaign_not_found(client, seeded_db): resp = await client.get("/api/campaigns/does-not-exist") assert resp.status_code == 404 diff --git a/tests/unit/test_campaign_report.py b/tests/unit/test_campaign_report.py index e5f4342..7d537e7 100644 --- a/tests/unit/test_campaign_report.py +++ b/tests/unit/test_campaign_report.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone import pytest from sqlmodel import Session, SQLModel, create_engine -from agenteval.evaluation.report import generate_campaign_report +from agenteval.evaluation.report import generate_campaign_report, summarize_campaign_progress from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus from agenteval.storage.repository import CampaignRepository, RunRepository @@ -151,3 +151,29 @@ def test_time_scale_only_affects_bucketing_not_numbers(report_session): assert _bucket(report, 6)["run_count"] == 1 assert _bucket(report, 6)["pass_rate"] == 0.8 assert _bucket(report, 6)["avg_latency_ms"] == 120.0 + + +def test_summarize_campaign_progress_uses_planned_total_and_pass_rate(report_session): + # plan totals 2 + 1 = 3 planned runs; only two have completed so far. + campaign = CampaignRepository(report_session).create(Campaign( + name="cycle", target_id="t-1", window_seconds=12, time_scale=1.0, + plan=[ + CampaignPlanEntry(scenario_id="s-a", offset_seconds=0, count=2), + CampaignPlanEntry(scenario_id="s-b", offset_seconds=6, count=1), + ], + )) + campaign.status = CampaignStatus.RUNNING + campaign.started_at = T0 + campaign = CampaignRepository(report_session).update(campaign) + + _seed_child(report_session, campaign.id, "s-a", RunStatus.COMPLETED, 0, pass_rate=1.0) + _seed_child(report_session, campaign.id, "s-a", RunStatus.COMPLETED, 1, pass_rate=0.0) + _seed_child(report_session, campaign.id, "s-b", RunStatus.FAILED, 6) # execution failure + + runs = RunRepository(report_session).list_by_campaign(campaign.id) + progress = summarize_campaign_progress(campaign, runs) + + assert progress["planned_total"] == 3 # Σ plan.count, not spawned-so-far + assert progress["completed_runs"] == 2 # FAILED does not count as completed + # ADR-0002: failed execution counts as 0.0 → (1.0 + 0.0 + 0.0) / 3 + assert progress["overall_pass_rate"] == round(1.0 / 3, 4)