feat(campaigns): live list progress, polling, and richer drill-down
Embed compact progress (completed/planned total + overall pass_rate, reusing the report's aggregation) into GET /campaigns so the list drops its N+1 report fetch. Poll list and open report drawer every 5s while the tab is active and a campaign is still running. Show scenario version and trigger source tags in the child-run drill-down.
This commit is contained in:
sinohqb
parent
c82532398b
commit
7ed765726f
@ -323,6 +323,22 @@ def _aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def summarize_campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict[str, Any]:
|
||||
"""Compact list-row progress: completed vs *planned* total, plus pass_rate.
|
||||
|
||||
Unlike ``campaign_progress`` (live window position), this powers the list
|
||||
view. ``planned_total`` is the sum of plan-entry counts — a fixed target the
|
||||
campaign works toward, so the progress bar fills from 0 rather than tracking
|
||||
a growing spawned count. ``overall_pass_rate`` reuses ``_aggregate_runs`` so
|
||||
the list figure matches the report exactly (ADR-0002: failures count as 0.0).
|
||||
"""
|
||||
return {
|
||||
"completed_runs": sum(1 for r in runs if r.status == RunStatus.COMPLETED),
|
||||
"planned_total": sum(entry.count for entry in campaign.plan),
|
||||
"overall_pass_rate": _aggregate_runs(runs)["pass_rate"],
|
||||
}
|
||||
|
||||
|
||||
def generate_campaign_report(
|
||||
campaign: Campaign,
|
||||
runs: list[EvalRun],
|
||||
|
||||
@ -12,7 +12,11 @@ from pydantic import BaseModel, Field
|
||||
from sqlmodel import Session
|
||||
|
||||
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
||||
from agenteval.evaluation.report import generate_campaign_report, render_campaign_markdown_report
|
||||
from agenteval.evaluation.report import (
|
||||
generate_campaign_report,
|
||||
render_campaign_markdown_report,
|
||||
summarize_campaign_progress,
|
||||
)
|
||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
|
||||
from agenteval.storage.db import utc_now
|
||||
from agenteval.storage.repository import (
|
||||
@ -36,7 +40,15 @@ class CreateCampaignRequest(BaseModel):
|
||||
|
||||
@router.get("")
|
||||
async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]:
|
||||
return [c.model_dump() for c in CampaignRepository(session).list_all()]
|
||||
repo = CampaignRepository(session)
|
||||
run_repo = RunRepository(session)
|
||||
result = []
|
||||
for campaign in repo.list_all():
|
||||
data = campaign.model_dump()
|
||||
runs = run_repo.list_by_campaign(campaign.id)
|
||||
data["progress"] = summarize_campaign_progress(campaign, runs)
|
||||
result.append(data)
|
||||
return result
|
||||
|
||||
|
||||
@router.post("")
|
||||
|
||||
@ -135,7 +135,7 @@ export interface ModelConfigReference {
|
||||
purpose: string
|
||||
}
|
||||
|
||||
export type RunTrigger = 'manual' | 'ai_assistant' | 'cli'
|
||||
export type RunTrigger = 'manual' | 'ai_assistant' | 'cli' | 'campaign'
|
||||
|
||||
export interface Run {
|
||||
id: string
|
||||
@ -298,6 +298,12 @@ export interface CampaignProgress {
|
||||
completed_runs: number
|
||||
}
|
||||
|
||||
export interface CampaignListProgress {
|
||||
completed_runs: number
|
||||
planned_total: number
|
||||
overall_pass_rate: number | null
|
||||
}
|
||||
|
||||
export interface Campaign {
|
||||
id: string
|
||||
name: string
|
||||
@ -312,6 +318,10 @@ export interface Campaign {
|
||||
progress?: CampaignProgress
|
||||
}
|
||||
|
||||
export interface CampaignListItem extends Omit<Campaign, 'progress'> {
|
||||
progress: CampaignListProgress
|
||||
}
|
||||
|
||||
export interface CampaignTrendBucket {
|
||||
bucket_index: number
|
||||
start_seconds: number
|
||||
@ -360,7 +370,7 @@ export interface CreateCampaignPayload {
|
||||
}
|
||||
|
||||
export const campaignsApi = {
|
||||
list: () => api.get<Campaign[]>('/campaigns'),
|
||||
list: () => api.get<CampaignListItem[]>('/campaigns'),
|
||||
get: (id: string) => api.get<Campaign>(`/campaigns/${id}`),
|
||||
create: (data: CreateCampaignPayload) => api.post<Campaign>('/campaigns', data),
|
||||
cancel: (id: string) => api.post<Campaign>(`/campaigns/${id}/cancel`),
|
||||
|
||||
@ -12,12 +12,13 @@ import { Line } from '@ant-design/charts'
|
||||
import PageWrapper from '../components/PageWrapper'
|
||||
import {
|
||||
campaignsApi, targetsApi, scenariosApi, runsApi,
|
||||
type Campaign, type CampaignReport, type Target, type Scenario, type Run,
|
||||
type CampaignListItem, type CampaignReport, type Target, type Scenario, type Run,
|
||||
} from '../api'
|
||||
import { passRateColor } from '../utils/colors'
|
||||
import { shortDateTime } from '../utils/date'
|
||||
import { useOnTabActive } from '../hooks/useOnTabActive'
|
||||
import { colors, statusColors, statusLabels } from '../tokens'
|
||||
import { useTabStore } from '../stores/tabStore'
|
||||
import { colors, statusColors, statusLabels, triggerColors, triggerLabels } from '../tokens'
|
||||
|
||||
const CAMPAIGN_STATUS: Record<string, { label: string; color: string }> = {
|
||||
planned: { label: '计划中', color: 'default' },
|
||||
@ -29,6 +30,10 @@ const CAMPAIGN_STATUS: Record<string, { label: string; color: string }> = {
|
||||
|
||||
const WINDOW_OPTIONS = [6, 12, 24, 48, 72].map((h) => ({ label: `${h} 小时`, value: h * 3600 }))
|
||||
|
||||
const POLL_INTERVAL_MS = 5000
|
||||
|
||||
const isActiveStatus = (status: string) => status === 'planned' || status === 'running'
|
||||
|
||||
function fmtWindow(seconds: number): string {
|
||||
if (seconds % 3600 === 0) return `${seconds / 3600}h`
|
||||
if (seconds % 60 === 0) return `${seconds / 60}m`
|
||||
@ -47,10 +52,10 @@ interface PlanFormEntry {
|
||||
|
||||
export default function CampaignsPage() {
|
||||
const navigate = useNavigate()
|
||||
const [campaigns, setCampaigns] = useState<Campaign[]>([])
|
||||
const activeKey = useTabStore((s) => s.activeKey)
|
||||
const [campaigns, setCampaigns] = useState<CampaignListItem[]>([])
|
||||
const [targets, setTargets] = useState<Target[]>([])
|
||||
const [scenarios, setScenarios] = useState<Scenario[]>([])
|
||||
const [reportMap, setReportMap] = useState<Record<string, CampaignReport>>({})
|
||||
const [loading, setLoading] = useState(false)
|
||||
|
||||
const [createOpen, setCreateOpen] = useState(false)
|
||||
@ -64,8 +69,8 @@ export default function CampaignsPage() {
|
||||
|
||||
const targetName = (id: string) => targets.find((t) => t.id === id)?.name ?? id.slice(0, 8)
|
||||
|
||||
const loadData = async () => {
|
||||
setLoading(true)
|
||||
const loadData = async (silent = false) => {
|
||||
if (!silent) setLoading(true)
|
||||
try {
|
||||
const [c, t, s] = await Promise.all([
|
||||
campaignsApi.list(), targetsApi.list(), scenariosApi.list(),
|
||||
@ -73,24 +78,23 @@ export default function CampaignsPage() {
|
||||
setCampaigns(c.data)
|
||||
setTargets(t.data)
|
||||
setScenarios(s.data)
|
||||
// Fetch each campaign's aggregate report so the list can show progress
|
||||
// and overall pass_rate at a glance (activity counts are small).
|
||||
const reports = await Promise.all(
|
||||
c.data.map((camp) =>
|
||||
campaignsApi.report(camp.id).then((r) => r.data).catch(() => null),
|
||||
),
|
||||
)
|
||||
const map: Record<string, CampaignReport> = {}
|
||||
reports.forEach((r) => { if (r) map[r.campaign_id] = r })
|
||||
setReportMap(map)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
if (!silent) setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => { loadData() }, [])
|
||||
useOnTabActive('/campaigns', loadData)
|
||||
|
||||
// Poll the list while this tab is active and a campaign is still working —
|
||||
// compressed dev-line campaigns change fast. Stop once all are terminal.
|
||||
const hasActiveCampaign = campaigns.some((c) => isActiveStatus(c.status))
|
||||
useEffect(() => {
|
||||
if (activeKey !== '/campaigns' || !hasActiveCampaign) return
|
||||
const id = setInterval(() => loadData(true), POLL_INTERVAL_MS)
|
||||
return () => clearInterval(id)
|
||||
}, [activeKey, hasActiveCampaign])
|
||||
|
||||
const openCreate = () => {
|
||||
form.setFieldsValue({
|
||||
name: '', target_id: undefined, window_seconds: 24 * 3600, time_scale: 1,
|
||||
@ -128,50 +132,63 @@ export default function CampaignsPage() {
|
||||
loadData()
|
||||
}
|
||||
|
||||
const openReport = async (campaign: Campaign) => {
|
||||
setReportOpen(true)
|
||||
setReportLoading(true)
|
||||
setReport(null)
|
||||
setReportRuns([])
|
||||
const fetchReport = async (campaignId: string, silent = false) => {
|
||||
if (!silent) {
|
||||
setReportLoading(true)
|
||||
setReport(null)
|
||||
setReportRuns([])
|
||||
}
|
||||
try {
|
||||
const [rep, runs] = await Promise.all([
|
||||
campaignsApi.report(campaign.id),
|
||||
campaignsApi.report(campaignId),
|
||||
runsApi.list(),
|
||||
])
|
||||
setReport(rep.data)
|
||||
setReportRuns(runs.data.filter((r) => r.campaign_id === campaign.id))
|
||||
setReportRuns(runs.data.filter((r) => r.campaign_id === campaignId))
|
||||
} finally {
|
||||
setReportLoading(false)
|
||||
if (!silent) setReportLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
const openReport = (campaign: CampaignListItem) => {
|
||||
setReportOpen(true)
|
||||
fetchReport(campaign.id)
|
||||
}
|
||||
|
||||
// Keep the open report drawer live while its campaign is still running.
|
||||
const reportId = report?.campaign_id ?? null
|
||||
const reportCampaignActive = campaigns.some(
|
||||
(c) => c.id === reportId && isActiveStatus(c.status),
|
||||
)
|
||||
useEffect(() => {
|
||||
if (activeKey !== '/campaigns' || !reportOpen || !reportId || !reportCampaignActive) return
|
||||
const id = setInterval(() => fetchReport(reportId, true), POLL_INTERVAL_MS)
|
||||
return () => clearInterval(id)
|
||||
}, [activeKey, reportOpen, reportId, reportCampaignActive])
|
||||
|
||||
const columns = [
|
||||
{ title: '名称', dataIndex: 'name', key: 'name' },
|
||||
{ title: '评测对象', key: 'target', render: (_: unknown, c: Campaign) => targetName(c.target_id) },
|
||||
{ title: '窗口', key: 'window', render: (_: unknown, c: Campaign) => fmtWindow(c.window_seconds) },
|
||||
{ title: '评测对象', key: 'target', render: (_: unknown, c: CampaignListItem) => targetName(c.target_id) },
|
||||
{ title: '窗口', key: 'window', render: (_: unknown, c: CampaignListItem) => fmtWindow(c.window_seconds) },
|
||||
{
|
||||
title: '倍速', key: 'scale',
|
||||
render: (_: unknown, c: Campaign) => (c.time_scale === 1 ? '实时' : `×${c.time_scale}`),
|
||||
render: (_: unknown, c: CampaignListItem) => (c.time_scale === 1 ? '实时' : `×${c.time_scale}`),
|
||||
},
|
||||
{
|
||||
title: '状态', key: 'status',
|
||||
render: (_: unknown, c: Campaign) => {
|
||||
render: (_: unknown, c: CampaignListItem) => {
|
||||
const meta = CAMPAIGN_STATUS[c.status] ?? { label: c.status, color: 'default' }
|
||||
return <Tag color={meta.color}>{meta.label}</Tag>
|
||||
},
|
||||
},
|
||||
{
|
||||
title: '进度', key: 'progress',
|
||||
render: (_: unknown, c: Campaign) => {
|
||||
const rep = reportMap[c.id]
|
||||
if (!rep) return '—'
|
||||
return `${rep.summary.completed_runs}/${rep.summary.total_runs}`
|
||||
},
|
||||
render: (_: unknown, c: CampaignListItem) => `${c.progress.completed_runs}/${c.progress.planned_total}`,
|
||||
},
|
||||
{
|
||||
title: '整窗通过率', key: 'pass_rate',
|
||||
render: (_: unknown, c: Campaign) => {
|
||||
const rate = reportMap[c.id]?.summary.overall_pass_rate
|
||||
render: (_: unknown, c: CampaignListItem) => {
|
||||
const rate = c.progress.overall_pass_rate
|
||||
if (rate == null) return '—'
|
||||
return (
|
||||
<Progress
|
||||
@ -183,10 +200,10 @@ export default function CampaignsPage() {
|
||||
},
|
||||
{
|
||||
title: '操作', key: 'actions',
|
||||
render: (_: unknown, c: Campaign) => (
|
||||
render: (_: unknown, c: CampaignListItem) => (
|
||||
<Space>
|
||||
<Button size="small" icon={<BarChartOutlined />} onClick={() => openReport(c)}>报告</Button>
|
||||
{(c.status === 'planned' || c.status === 'running') && (
|
||||
{isActiveStatus(c.status) && (
|
||||
<Popconfirm title="取消该活动？已完成的子运行会保留。" onConfirm={() => cancelCampaign(c.id)}>
|
||||
<Button size="small" danger icon={<StopOutlined />}>取消</Button>
|
||||
</Popconfirm>
|
||||
@ -243,7 +260,19 @@ export default function CampaignsPage() {
|
||||
{ title: '子运行', dataIndex: 'id', key: 'id', render: (id: string) => <code>{id.slice(0, 8)}</code> },
|
||||
{
|
||||
title: '场景', key: 'scenario',
|
||||
render: (_: unknown, r: Run) => scenarios.find((s) => s.id === r.scenario_id)?.name ?? r.scenario_id.slice(0, 8),
|
||||
render: (_: unknown, r: Run) => (
|
||||
<Space size={4}>
|
||||
<span>{scenarios.find((s) => s.id === r.scenario_id)?.name ?? r.scenario_id.slice(0, 8)}</span>
|
||||
{r.scenario_version != null && <Tag>v{r.scenario_version}</Tag>}
|
||||
</Space>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: '来源', key: 'trigger',
|
||||
render: (_: unknown, r: Run) => {
|
||||
const t = r.triggered_by ?? ''
|
||||
return <Tag color={triggerColors[t] ?? 'default'}>{triggerLabels[t] ?? t}</Tag>
|
||||
},
|
||||
},
|
||||
{
|
||||
title: '状态', key: 'status',
|
||||
@ -277,7 +306,7 @@ export default function CampaignsPage() {
|
||||
fullHeight
|
||||
extra={
|
||||
<Space>
|
||||
<Button icon={<ReloadOutlined />} onClick={loadData} />
|
||||
<Button icon={<ReloadOutlined />} onClick={() => loadData()} />
|
||||
<Button type="primary" icon={<PlusOutlined />} onClick={openCreate}>新建活动</Button>
|
||||
</Space>
|
||||
}
|
||||
|
||||
@ -47,12 +47,14 @@ export const triggerLabels: Record<string, string> = {
|
||||
manual: '手动',
|
||||
ai_assistant: 'AI 助手',
|
||||
cli: 'CLI',
|
||||
campaign: '活动',
|
||||
}
|
||||
|
||||
export const triggerColors: Record<string, string> = {
|
||||
manual: 'default',
|
||||
ai_assistant: 'purple',
|
||||
cli: 'blue',
|
||||
campaign: 'cyan',
|
||||
}
|
||||
|
||||
export const spacing = {
|
||||
|
||||
@ -132,6 +132,31 @@ async def test_list_campaigns_after_create(client, seeded_db):
|
||||
assert listing[0]["name"] == "24h-cycle"
|
||||
|
||||
|
||||
async def test_list_campaigns_embeds_progress(client, seeded_db):
|
||||
from agenteval.models import EvalRun, RunStatus
|
||||
|
||||
# _valid_payload plan totals 2 + 1 = 3 planned runs.
|
||||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||||
repo = RunRepository(seeded_db)
|
||||
repo.create(EvalRun(
|
||||
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||||
status=RunStatus.COMPLETED,
|
||||
summary={"pass_rate": 1.0, "avg_latency_ms": 100},
|
||||
))
|
||||
repo.create(EvalRun(
|
||||
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||||
status=RunStatus.COMPLETED,
|
||||
summary={"pass_rate": 0.0, "avg_latency_ms": 200},
|
||||
))
|
||||
|
||||
listing = (await client.get("/api/campaigns")).json()
|
||||
assert len(listing) == 1
|
||||
progress = listing[0]["progress"]
|
||||
assert progress["planned_total"] == 3 # Σ plan.count
|
||||
assert progress["completed_runs"] == 2
|
||||
assert progress["overall_pass_rate"] == 0.5
|
||||
|
||||
|
||||
async def test_get_campaign_not_found(client, seeded_db):
|
||||
resp = await client.get("/api/campaigns/does-not-exist")
|
||||
assert resp.status_code == 404
|
||||
|
||||
@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone
|
||||
import pytest
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
from agenteval.evaluation.report import generate_campaign_report
|
||||
from agenteval.evaluation.report import generate_campaign_report, summarize_campaign_progress
|
||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus
|
||||
from agenteval.storage.repository import CampaignRepository, RunRepository
|
||||
|
||||
@ -151,3 +151,29 @@ def test_time_scale_only_affects_bucketing_not_numbers(report_session):
|
||||
assert _bucket(report, 6)["run_count"] == 1
|
||||
assert _bucket(report, 6)["pass_rate"] == 0.8
|
||||
assert _bucket(report, 6)["avg_latency_ms"] == 120.0
|
||||
|
||||
|
||||
def test_summarize_campaign_progress_uses_planned_total_and_pass_rate(report_session):
|
||||
# plan totals 2 + 1 = 3 planned runs; only two have completed so far.
|
||||
campaign = CampaignRepository(report_session).create(Campaign(
|
||||
name="cycle", target_id="t-1", window_seconds=12, time_scale=1.0,
|
||||
plan=[
|
||||
CampaignPlanEntry(scenario_id="s-a", offset_seconds=0, count=2),
|
||||
CampaignPlanEntry(scenario_id="s-b", offset_seconds=6, count=1),
|
||||
],
|
||||
))
|
||||
campaign.status = CampaignStatus.RUNNING
|
||||
campaign.started_at = T0
|
||||
campaign = CampaignRepository(report_session).update(campaign)
|
||||
|
||||
_seed_child(report_session, campaign.id, "s-a", RunStatus.COMPLETED, 0, pass_rate=1.0)
|
||||
_seed_child(report_session, campaign.id, "s-a", RunStatus.COMPLETED, 1, pass_rate=0.0)
|
||||
_seed_child(report_session, campaign.id, "s-b", RunStatus.FAILED, 6) # execution failure
|
||||
|
||||
runs = RunRepository(report_session).list_by_campaign(campaign.id)
|
||||
progress = summarize_campaign_progress(campaign, runs)
|
||||
|
||||
assert progress["planned_total"] == 3 # Σ plan.count, not spawned-so-far
|
||||
assert progress["completed_runs"] == 2 # FAILED does not count as completed
|
||||
# ADR-0002: failed execution counts as 0.0 → (1.0 + 0.0 + 0.0) / 3
|
||||
assert progress["overall_pass_rate"] == round(1.0 / 3, 4)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user