feat(campaigns): live list progress, polling, and richer drill-down

Embed compact progress (completed/planned total + overall pass_rate,
reusing the report's aggregation) into GET /campaigns so the list drops
its N+1 report fetch. Poll list and open report drawer every 5s while the
tab is active and a campaign is still running. Show scenario version and
trigger source tags in the child-run drill-down.
This commit is contained in:
sinohqb 2026-07-30 15:35:18 +08:00
parent c82532398b
commit 7ed765726f
7 changed files with 165 additions and 45 deletions

View File

@ -323,6 +323,22 @@ def _aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]:
} }
def summarize_campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict[str, Any]:
"""Compact list-row progress: completed vs *planned* total, plus pass_rate.
Unlike ``campaign_progress`` (live window position), this powers the list
view. ``planned_total`` is the sum of plan-entry counts a fixed target the
campaign works toward, so the progress bar fills from 0 rather than tracking
a growing spawned count. ``overall_pass_rate`` reuses ``_aggregate_runs`` so
the list figure matches the report exactly (ADR-0002: failures count as 0.0).
"""
return {
"completed_runs": sum(1 for r in runs if r.status == RunStatus.COMPLETED),
"planned_total": sum(entry.count for entry in campaign.plan),
"overall_pass_rate": _aggregate_runs(runs)["pass_rate"],
}
def generate_campaign_report( def generate_campaign_report(
campaign: Campaign, campaign: Campaign,
runs: list[EvalRun], runs: list[EvalRun],

View File

@ -12,7 +12,11 @@ from pydantic import BaseModel, Field
from sqlmodel import Session from sqlmodel import Session
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
from agenteval.evaluation.report import generate_campaign_report, render_campaign_markdown_report from agenteval.evaluation.report import (
generate_campaign_report,
render_campaign_markdown_report,
summarize_campaign_progress,
)
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
from agenteval.storage.db import utc_now from agenteval.storage.db import utc_now
from agenteval.storage.repository import ( from agenteval.storage.repository import (
@ -36,7 +40,15 @@ class CreateCampaignRequest(BaseModel):
@router.get("") @router.get("")
async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]: async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]:
return [c.model_dump() for c in CampaignRepository(session).list_all()] repo = CampaignRepository(session)
run_repo = RunRepository(session)
result = []
for campaign in repo.list_all():
data = campaign.model_dump()
runs = run_repo.list_by_campaign(campaign.id)
data["progress"] = summarize_campaign_progress(campaign, runs)
result.append(data)
return result
@router.post("") @router.post("")

View File

@ -135,7 +135,7 @@ export interface ModelConfigReference {
purpose: string purpose: string
} }
export type RunTrigger = 'manual' | 'ai_assistant' | 'cli' export type RunTrigger = 'manual' | 'ai_assistant' | 'cli' | 'campaign'
export interface Run { export interface Run {
id: string id: string
@ -298,6 +298,12 @@ export interface CampaignProgress {
completed_runs: number completed_runs: number
} }
export interface CampaignListProgress {
completed_runs: number
planned_total: number
overall_pass_rate: number | null
}
export interface Campaign { export interface Campaign {
id: string id: string
name: string name: string
@ -312,6 +318,10 @@ export interface Campaign {
progress?: CampaignProgress progress?: CampaignProgress
} }
export interface CampaignListItem extends Omit<Campaign, 'progress'> {
progress: CampaignListProgress
}
export interface CampaignTrendBucket { export interface CampaignTrendBucket {
bucket_index: number bucket_index: number
start_seconds: number start_seconds: number
@ -360,7 +370,7 @@ export interface CreateCampaignPayload {
} }
export const campaignsApi = { export const campaignsApi = {
list: () => api.get<Campaign[]>('/campaigns'), list: () => api.get<CampaignListItem[]>('/campaigns'),
get: (id: string) => api.get<Campaign>(`/campaigns/${id}`), get: (id: string) => api.get<Campaign>(`/campaigns/${id}`),
create: (data: CreateCampaignPayload) => api.post<Campaign>('/campaigns', data), create: (data: CreateCampaignPayload) => api.post<Campaign>('/campaigns', data),
cancel: (id: string) => api.post<Campaign>(`/campaigns/${id}/cancel`), cancel: (id: string) => api.post<Campaign>(`/campaigns/${id}/cancel`),

View File

@ -12,12 +12,13 @@ import { Line } from '@ant-design/charts'
import PageWrapper from '../components/PageWrapper' import PageWrapper from '../components/PageWrapper'
import { import {
campaignsApi, targetsApi, scenariosApi, runsApi, campaignsApi, targetsApi, scenariosApi, runsApi,
type Campaign, type CampaignReport, type Target, type Scenario, type Run, type CampaignListItem, type CampaignReport, type Target, type Scenario, type Run,
} from '../api' } from '../api'
import { passRateColor } from '../utils/colors' import { passRateColor } from '../utils/colors'
import { shortDateTime } from '../utils/date' import { shortDateTime } from '../utils/date'
import { useOnTabActive } from '../hooks/useOnTabActive' import { useOnTabActive } from '../hooks/useOnTabActive'
import { colors, statusColors, statusLabels } from '../tokens' import { useTabStore } from '../stores/tabStore'
import { colors, statusColors, statusLabels, triggerColors, triggerLabels } from '../tokens'
const CAMPAIGN_STATUS: Record<string, { label: string; color: string }> = { const CAMPAIGN_STATUS: Record<string, { label: string; color: string }> = {
planned: { label: '计划中', color: 'default' }, planned: { label: '计划中', color: 'default' },
@ -29,6 +30,10 @@ const CAMPAIGN_STATUS: Record<string, { label: string; color: string }> = {
const WINDOW_OPTIONS = [6, 12, 24, 48, 72].map((h) => ({ label: `${h} 小时`, value: h * 3600 })) const WINDOW_OPTIONS = [6, 12, 24, 48, 72].map((h) => ({ label: `${h} 小时`, value: h * 3600 }))
const POLL_INTERVAL_MS = 5000
const isActiveStatus = (status: string) => status === 'planned' || status === 'running'
function fmtWindow(seconds: number): string { function fmtWindow(seconds: number): string {
if (seconds % 3600 === 0) return `${seconds / 3600}h` if (seconds % 3600 === 0) return `${seconds / 3600}h`
if (seconds % 60 === 0) return `${seconds / 60}m` if (seconds % 60 === 0) return `${seconds / 60}m`
@ -47,10 +52,10 @@ interface PlanFormEntry {
export default function CampaignsPage() { export default function CampaignsPage() {
const navigate = useNavigate() const navigate = useNavigate()
const [campaigns, setCampaigns] = useState<Campaign[]>([]) const activeKey = useTabStore((s) => s.activeKey)
const [campaigns, setCampaigns] = useState<CampaignListItem[]>([])
const [targets, setTargets] = useState<Target[]>([]) const [targets, setTargets] = useState<Target[]>([])
const [scenarios, setScenarios] = useState<Scenario[]>([]) const [scenarios, setScenarios] = useState<Scenario[]>([])
const [reportMap, setReportMap] = useState<Record<string, CampaignReport>>({})
const [loading, setLoading] = useState(false) const [loading, setLoading] = useState(false)
const [createOpen, setCreateOpen] = useState(false) const [createOpen, setCreateOpen] = useState(false)
@ -64,8 +69,8 @@ export default function CampaignsPage() {
const targetName = (id: string) => targets.find((t) => t.id === id)?.name ?? id.slice(0, 8) const targetName = (id: string) => targets.find((t) => t.id === id)?.name ?? id.slice(0, 8)
const loadData = async () => { const loadData = async (silent = false) => {
setLoading(true) if (!silent) setLoading(true)
try { try {
const [c, t, s] = await Promise.all([ const [c, t, s] = await Promise.all([
campaignsApi.list(), targetsApi.list(), scenariosApi.list(), campaignsApi.list(), targetsApi.list(), scenariosApi.list(),
@ -73,24 +78,23 @@ export default function CampaignsPage() {
setCampaigns(c.data) setCampaigns(c.data)
setTargets(t.data) setTargets(t.data)
setScenarios(s.data) setScenarios(s.data)
// Fetch each campaign's aggregate report so the list can show progress
// and overall pass_rate at a glance (activity counts are small).
const reports = await Promise.all(
c.data.map((camp) =>
campaignsApi.report(camp.id).then((r) => r.data).catch(() => null),
),
)
const map: Record<string, CampaignReport> = {}
reports.forEach((r) => { if (r) map[r.campaign_id] = r })
setReportMap(map)
} finally { } finally {
setLoading(false) if (!silent) setLoading(false)
} }
} }
useEffect(() => { loadData() }, []) useEffect(() => { loadData() }, [])
useOnTabActive('/campaigns', loadData) useOnTabActive('/campaigns', loadData)
// Poll the list while this tab is active and a campaign is still working —
// compressed dev-line campaigns change fast. Stop once all are terminal.
const hasActiveCampaign = campaigns.some((c) => isActiveStatus(c.status))
useEffect(() => {
if (activeKey !== '/campaigns' || !hasActiveCampaign) return
const id = setInterval(() => loadData(true), POLL_INTERVAL_MS)
return () => clearInterval(id)
}, [activeKey, hasActiveCampaign])
const openCreate = () => { const openCreate = () => {
form.setFieldsValue({ form.setFieldsValue({
name: '', target_id: undefined, window_seconds: 24 * 3600, time_scale: 1, name: '', target_id: undefined, window_seconds: 24 * 3600, time_scale: 1,
@ -128,50 +132,63 @@ export default function CampaignsPage() {
loadData() loadData()
} }
const openReport = async (campaign: Campaign) => { const fetchReport = async (campaignId: string, silent = false) => {
setReportOpen(true) if (!silent) {
setReportLoading(true) setReportLoading(true)
setReport(null) setReport(null)
setReportRuns([]) setReportRuns([])
}
try { try {
const [rep, runs] = await Promise.all([ const [rep, runs] = await Promise.all([
campaignsApi.report(campaign.id), campaignsApi.report(campaignId),
runsApi.list(), runsApi.list(),
]) ])
setReport(rep.data) setReport(rep.data)
setReportRuns(runs.data.filter((r) => r.campaign_id === campaign.id)) setReportRuns(runs.data.filter((r) => r.campaign_id === campaignId))
} finally { } finally {
setReportLoading(false) if (!silent) setReportLoading(false)
} }
} }
const openReport = (campaign: CampaignListItem) => {
setReportOpen(true)
fetchReport(campaign.id)
}
// Keep the open report drawer live while its campaign is still running.
const reportId = report?.campaign_id ?? null
const reportCampaignActive = campaigns.some(
(c) => c.id === reportId && isActiveStatus(c.status),
)
useEffect(() => {
if (activeKey !== '/campaigns' || !reportOpen || !reportId || !reportCampaignActive) return
const id = setInterval(() => fetchReport(reportId, true), POLL_INTERVAL_MS)
return () => clearInterval(id)
}, [activeKey, reportOpen, reportId, reportCampaignActive])
const columns = [ const columns = [
{ title: '名称', dataIndex: 'name', key: 'name' }, { title: '名称', dataIndex: 'name', key: 'name' },
{ title: '评测对象', key: 'target', render: (_: unknown, c: Campaign) => targetName(c.target_id) }, { title: '评测对象', key: 'target', render: (_: unknown, c: CampaignListItem) => targetName(c.target_id) },
{ title: '窗口', key: 'window', render: (_: unknown, c: Campaign) => fmtWindow(c.window_seconds) }, { title: '窗口', key: 'window', render: (_: unknown, c: CampaignListItem) => fmtWindow(c.window_seconds) },
{ {
title: '倍速', key: 'scale', title: '倍速', key: 'scale',
render: (_: unknown, c: Campaign) => (c.time_scale === 1 ? '实时' : `×${c.time_scale}`), render: (_: unknown, c: CampaignListItem) => (c.time_scale === 1 ? '实时' : `×${c.time_scale}`),
}, },
{ {
title: '状态', key: 'status', title: '状态', key: 'status',
render: (_: unknown, c: Campaign) => { render: (_: unknown, c: CampaignListItem) => {
const meta = CAMPAIGN_STATUS[c.status] ?? { label: c.status, color: 'default' } const meta = CAMPAIGN_STATUS[c.status] ?? { label: c.status, color: 'default' }
return <Tag color={meta.color}>{meta.label}</Tag> return <Tag color={meta.color}>{meta.label}</Tag>
}, },
}, },
{ {
title: '进度', key: 'progress', title: '进度', key: 'progress',
render: (_: unknown, c: Campaign) => { render: (_: unknown, c: CampaignListItem) => `${c.progress.completed_runs}/${c.progress.planned_total}`,
const rep = reportMap[c.id]
if (!rep) return '—'
return `${rep.summary.completed_runs}/${rep.summary.total_runs}`
},
}, },
{ {
title: '整窗通过率', key: 'pass_rate', title: '整窗通过率', key: 'pass_rate',
render: (_: unknown, c: Campaign) => { render: (_: unknown, c: CampaignListItem) => {
const rate = reportMap[c.id]?.summary.overall_pass_rate const rate = c.progress.overall_pass_rate
if (rate == null) return '—' if (rate == null) return '—'
return ( return (
<Progress <Progress
@ -183,10 +200,10 @@ export default function CampaignsPage() {
}, },
{ {
title: '操作', key: 'actions', title: '操作', key: 'actions',
render: (_: unknown, c: Campaign) => ( render: (_: unknown, c: CampaignListItem) => (
<Space> <Space>
<Button size="small" icon={<BarChartOutlined />} onClick={() => openReport(c)}></Button> <Button size="small" icon={<BarChartOutlined />} onClick={() => openReport(c)}></Button>
{(c.status === 'planned' || c.status === 'running') && ( {isActiveStatus(c.status) && (
<Popconfirm title="取消该活动?已完成的子运行会保留。" onConfirm={() => cancelCampaign(c.id)}> <Popconfirm title="取消该活动?已完成的子运行会保留。" onConfirm={() => cancelCampaign(c.id)}>
<Button size="small" danger icon={<StopOutlined />}></Button> <Button size="small" danger icon={<StopOutlined />}></Button>
</Popconfirm> </Popconfirm>
@ -243,7 +260,19 @@ export default function CampaignsPage() {
{ title: '子运行', dataIndex: 'id', key: 'id', render: (id: string) => <code>{id.slice(0, 8)}</code> }, { title: '子运行', dataIndex: 'id', key: 'id', render: (id: string) => <code>{id.slice(0, 8)}</code> },
{ {
title: '场景', key: 'scenario', title: '场景', key: 'scenario',
render: (_: unknown, r: Run) => scenarios.find((s) => s.id === r.scenario_id)?.name ?? r.scenario_id.slice(0, 8), render: (_: unknown, r: Run) => (
<Space size={4}>
<span>{scenarios.find((s) => s.id === r.scenario_id)?.name ?? r.scenario_id.slice(0, 8)}</span>
{r.scenario_version != null && <Tag>v{r.scenario_version}</Tag>}
</Space>
),
},
{
title: '来源', key: 'trigger',
render: (_: unknown, r: Run) => {
const t = r.triggered_by ?? ''
return <Tag color={triggerColors[t] ?? 'default'}>{triggerLabels[t] ?? t}</Tag>
},
}, },
{ {
title: '状态', key: 'status', title: '状态', key: 'status',
@ -277,7 +306,7 @@ export default function CampaignsPage() {
fullHeight fullHeight
extra={ extra={
<Space> <Space>
<Button icon={<ReloadOutlined />} onClick={loadData} /> <Button icon={<ReloadOutlined />} onClick={() => loadData()} />
<Button type="primary" icon={<PlusOutlined />} onClick={openCreate}></Button> <Button type="primary" icon={<PlusOutlined />} onClick={openCreate}></Button>
</Space> </Space>
} }

View File

@ -47,12 +47,14 @@ export const triggerLabels: Record<string, string> = {
manual: '手动', manual: '手动',
ai_assistant: 'AI 助手', ai_assistant: 'AI 助手',
cli: 'CLI', cli: 'CLI',
campaign: '活动',
} }
export const triggerColors: Record<string, string> = { export const triggerColors: Record<string, string> = {
manual: 'default', manual: 'default',
ai_assistant: 'purple', ai_assistant: 'purple',
cli: 'blue', cli: 'blue',
campaign: 'cyan',
} }
export const spacing = { export const spacing = {

View File

@ -132,6 +132,31 @@ async def test_list_campaigns_after_create(client, seeded_db):
assert listing[0]["name"] == "24h-cycle" assert listing[0]["name"] == "24h-cycle"
async def test_list_campaigns_embeds_progress(client, seeded_db):
from agenteval.models import EvalRun, RunStatus
# _valid_payload plan totals 2 + 1 = 3 planned runs.
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
repo = RunRepository(seeded_db)
repo.create(EvalRun(
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
status=RunStatus.COMPLETED,
summary={"pass_rate": 1.0, "avg_latency_ms": 100},
))
repo.create(EvalRun(
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
status=RunStatus.COMPLETED,
summary={"pass_rate": 0.0, "avg_latency_ms": 200},
))
listing = (await client.get("/api/campaigns")).json()
assert len(listing) == 1
progress = listing[0]["progress"]
assert progress["planned_total"] == 3 # Σ plan.count
assert progress["completed_runs"] == 2
assert progress["overall_pass_rate"] == 0.5
async def test_get_campaign_not_found(client, seeded_db): async def test_get_campaign_not_found(client, seeded_db):
resp = await client.get("/api/campaigns/does-not-exist") resp = await client.get("/api/campaigns/does-not-exist")
assert resp.status_code == 404 assert resp.status_code == 404

View File

@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone
import pytest import pytest
from sqlmodel import Session, SQLModel, create_engine from sqlmodel import Session, SQLModel, create_engine
from agenteval.evaluation.report import generate_campaign_report from agenteval.evaluation.report import generate_campaign_report, summarize_campaign_progress
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus
from agenteval.storage.repository import CampaignRepository, RunRepository from agenteval.storage.repository import CampaignRepository, RunRepository
@ -151,3 +151,29 @@ def test_time_scale_only_affects_bucketing_not_numbers(report_session):
assert _bucket(report, 6)["run_count"] == 1 assert _bucket(report, 6)["run_count"] == 1
assert _bucket(report, 6)["pass_rate"] == 0.8 assert _bucket(report, 6)["pass_rate"] == 0.8
assert _bucket(report, 6)["avg_latency_ms"] == 120.0 assert _bucket(report, 6)["avg_latency_ms"] == 120.0
def test_summarize_campaign_progress_uses_planned_total_and_pass_rate(report_session):
# plan totals 2 + 1 = 3 planned runs; only two have completed so far.
campaign = CampaignRepository(report_session).create(Campaign(
name="cycle", target_id="t-1", window_seconds=12, time_scale=1.0,
plan=[
CampaignPlanEntry(scenario_id="s-a", offset_seconds=0, count=2),
CampaignPlanEntry(scenario_id="s-b", offset_seconds=6, count=1),
],
))
campaign.status = CampaignStatus.RUNNING
campaign.started_at = T0
campaign = CampaignRepository(report_session).update(campaign)
_seed_child(report_session, campaign.id, "s-a", RunStatus.COMPLETED, 0, pass_rate=1.0)
_seed_child(report_session, campaign.id, "s-a", RunStatus.COMPLETED, 1, pass_rate=0.0)
_seed_child(report_session, campaign.id, "s-b", RunStatus.FAILED, 6) # execution failure
runs = RunRepository(report_session).list_by_campaign(campaign.id)
progress = summarize_campaign_progress(campaign, runs)
assert progress["planned_total"] == 3 # Σ plan.count, not spawned-so-far
assert progress["completed_runs"] == 2 # FAILED does not count as completed
# ADR-0002: failed execution counts as 0.0 → (1.0 + 0.0 + 0.0) / 3
assert progress["overall_pass_rate"] == round(1.0 / 3, 4)