feat(campaigns): live list progress, polling, and richer drill-down
Embed compact progress (completed/planned total + overall pass_rate, reusing the report's aggregation) into GET /campaigns so the list drops its N+1 report fetch. Poll list and open report drawer every 5s while the tab is active and a campaign is still running. Show scenario version and trigger source tags in the child-run drill-down.
This commit is contained in:
parent
c82532398b
commit
7ed765726f
@ -323,6 +323,22 @@ def _aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def summarize_campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict[str, Any]:
|
||||||
|
"""Compact list-row progress: completed vs *planned* total, plus pass_rate.
|
||||||
|
|
||||||
|
Unlike ``campaign_progress`` (live window position), this powers the list
|
||||||
|
view. ``planned_total`` is the sum of plan-entry counts — a fixed target the
|
||||||
|
campaign works toward, so the progress bar fills from 0 rather than tracking
|
||||||
|
a growing spawned count. ``overall_pass_rate`` reuses ``_aggregate_runs`` so
|
||||||
|
the list figure matches the report exactly (ADR-0002: failures count as 0.0).
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"completed_runs": sum(1 for r in runs if r.status == RunStatus.COMPLETED),
|
||||||
|
"planned_total": sum(entry.count for entry in campaign.plan),
|
||||||
|
"overall_pass_rate": _aggregate_runs(runs)["pass_rate"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def generate_campaign_report(
|
def generate_campaign_report(
|
||||||
campaign: Campaign,
|
campaign: Campaign,
|
||||||
runs: list[EvalRun],
|
runs: list[EvalRun],
|
||||||
|
|||||||
@ -12,7 +12,11 @@ from pydantic import BaseModel, Field
|
|||||||
from sqlmodel import Session
|
from sqlmodel import Session
|
||||||
|
|
||||||
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
||||||
from agenteval.evaluation.report import generate_campaign_report, render_campaign_markdown_report
|
from agenteval.evaluation.report import (
|
||||||
|
generate_campaign_report,
|
||||||
|
render_campaign_markdown_report,
|
||||||
|
summarize_campaign_progress,
|
||||||
|
)
|
||||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
|
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
|
||||||
from agenteval.storage.db import utc_now
|
from agenteval.storage.db import utc_now
|
||||||
from agenteval.storage.repository import (
|
from agenteval.storage.repository import (
|
||||||
@ -36,7 +40,15 @@ class CreateCampaignRequest(BaseModel):
|
|||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]:
|
async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]:
|
||||||
return [c.model_dump() for c in CampaignRepository(session).list_all()]
|
repo = CampaignRepository(session)
|
||||||
|
run_repo = RunRepository(session)
|
||||||
|
result = []
|
||||||
|
for campaign in repo.list_all():
|
||||||
|
data = campaign.model_dump()
|
||||||
|
runs = run_repo.list_by_campaign(campaign.id)
|
||||||
|
data["progress"] = summarize_campaign_progress(campaign, runs)
|
||||||
|
result.append(data)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
@router.post("")
|
@router.post("")
|
||||||
|
|||||||
@ -135,7 +135,7 @@ export interface ModelConfigReference {
|
|||||||
purpose: string
|
purpose: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export type RunTrigger = 'manual' | 'ai_assistant' | 'cli'
|
export type RunTrigger = 'manual' | 'ai_assistant' | 'cli' | 'campaign'
|
||||||
|
|
||||||
export interface Run {
|
export interface Run {
|
||||||
id: string
|
id: string
|
||||||
@ -298,6 +298,12 @@ export interface CampaignProgress {
|
|||||||
completed_runs: number
|
completed_runs: number
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface CampaignListProgress {
|
||||||
|
completed_runs: number
|
||||||
|
planned_total: number
|
||||||
|
overall_pass_rate: number | null
|
||||||
|
}
|
||||||
|
|
||||||
export interface Campaign {
|
export interface Campaign {
|
||||||
id: string
|
id: string
|
||||||
name: string
|
name: string
|
||||||
@ -312,6 +318,10 @@ export interface Campaign {
|
|||||||
progress?: CampaignProgress
|
progress?: CampaignProgress
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface CampaignListItem extends Omit<Campaign, 'progress'> {
|
||||||
|
progress: CampaignListProgress
|
||||||
|
}
|
||||||
|
|
||||||
export interface CampaignTrendBucket {
|
export interface CampaignTrendBucket {
|
||||||
bucket_index: number
|
bucket_index: number
|
||||||
start_seconds: number
|
start_seconds: number
|
||||||
@ -360,7 +370,7 @@ export interface CreateCampaignPayload {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export const campaignsApi = {
|
export const campaignsApi = {
|
||||||
list: () => api.get<Campaign[]>('/campaigns'),
|
list: () => api.get<CampaignListItem[]>('/campaigns'),
|
||||||
get: (id: string) => api.get<Campaign>(`/campaigns/${id}`),
|
get: (id: string) => api.get<Campaign>(`/campaigns/${id}`),
|
||||||
create: (data: CreateCampaignPayload) => api.post<Campaign>('/campaigns', data),
|
create: (data: CreateCampaignPayload) => api.post<Campaign>('/campaigns', data),
|
||||||
cancel: (id: string) => api.post<Campaign>(`/campaigns/${id}/cancel`),
|
cancel: (id: string) => api.post<Campaign>(`/campaigns/${id}/cancel`),
|
||||||
|
|||||||
@ -12,12 +12,13 @@ import { Line } from '@ant-design/charts'
|
|||||||
import PageWrapper from '../components/PageWrapper'
|
import PageWrapper from '../components/PageWrapper'
|
||||||
import {
|
import {
|
||||||
campaignsApi, targetsApi, scenariosApi, runsApi,
|
campaignsApi, targetsApi, scenariosApi, runsApi,
|
||||||
type Campaign, type CampaignReport, type Target, type Scenario, type Run,
|
type CampaignListItem, type CampaignReport, type Target, type Scenario, type Run,
|
||||||
} from '../api'
|
} from '../api'
|
||||||
import { passRateColor } from '../utils/colors'
|
import { passRateColor } from '../utils/colors'
|
||||||
import { shortDateTime } from '../utils/date'
|
import { shortDateTime } from '../utils/date'
|
||||||
import { useOnTabActive } from '../hooks/useOnTabActive'
|
import { useOnTabActive } from '../hooks/useOnTabActive'
|
||||||
import { colors, statusColors, statusLabels } from '../tokens'
|
import { useTabStore } from '../stores/tabStore'
|
||||||
|
import { colors, statusColors, statusLabels, triggerColors, triggerLabels } from '../tokens'
|
||||||
|
|
||||||
const CAMPAIGN_STATUS: Record<string, { label: string; color: string }> = {
|
const CAMPAIGN_STATUS: Record<string, { label: string; color: string }> = {
|
||||||
planned: { label: '计划中', color: 'default' },
|
planned: { label: '计划中', color: 'default' },
|
||||||
@ -29,6 +30,10 @@ const CAMPAIGN_STATUS: Record<string, { label: string; color: string }> = {
|
|||||||
|
|
||||||
const WINDOW_OPTIONS = [6, 12, 24, 48, 72].map((h) => ({ label: `${h} 小时`, value: h * 3600 }))
|
const WINDOW_OPTIONS = [6, 12, 24, 48, 72].map((h) => ({ label: `${h} 小时`, value: h * 3600 }))
|
||||||
|
|
||||||
|
const POLL_INTERVAL_MS = 5000
|
||||||
|
|
||||||
|
const isActiveStatus = (status: string) => status === 'planned' || status === 'running'
|
||||||
|
|
||||||
function fmtWindow(seconds: number): string {
|
function fmtWindow(seconds: number): string {
|
||||||
if (seconds % 3600 === 0) return `${seconds / 3600}h`
|
if (seconds % 3600 === 0) return `${seconds / 3600}h`
|
||||||
if (seconds % 60 === 0) return `${seconds / 60}m`
|
if (seconds % 60 === 0) return `${seconds / 60}m`
|
||||||
@ -47,10 +52,10 @@ interface PlanFormEntry {
|
|||||||
|
|
||||||
export default function CampaignsPage() {
|
export default function CampaignsPage() {
|
||||||
const navigate = useNavigate()
|
const navigate = useNavigate()
|
||||||
const [campaigns, setCampaigns] = useState<Campaign[]>([])
|
const activeKey = useTabStore((s) => s.activeKey)
|
||||||
|
const [campaigns, setCampaigns] = useState<CampaignListItem[]>([])
|
||||||
const [targets, setTargets] = useState<Target[]>([])
|
const [targets, setTargets] = useState<Target[]>([])
|
||||||
const [scenarios, setScenarios] = useState<Scenario[]>([])
|
const [scenarios, setScenarios] = useState<Scenario[]>([])
|
||||||
const [reportMap, setReportMap] = useState<Record<string, CampaignReport>>({})
|
|
||||||
const [loading, setLoading] = useState(false)
|
const [loading, setLoading] = useState(false)
|
||||||
|
|
||||||
const [createOpen, setCreateOpen] = useState(false)
|
const [createOpen, setCreateOpen] = useState(false)
|
||||||
@ -64,8 +69,8 @@ export default function CampaignsPage() {
|
|||||||
|
|
||||||
const targetName = (id: string) => targets.find((t) => t.id === id)?.name ?? id.slice(0, 8)
|
const targetName = (id: string) => targets.find((t) => t.id === id)?.name ?? id.slice(0, 8)
|
||||||
|
|
||||||
const loadData = async () => {
|
const loadData = async (silent = false) => {
|
||||||
setLoading(true)
|
if (!silent) setLoading(true)
|
||||||
try {
|
try {
|
||||||
const [c, t, s] = await Promise.all([
|
const [c, t, s] = await Promise.all([
|
||||||
campaignsApi.list(), targetsApi.list(), scenariosApi.list(),
|
campaignsApi.list(), targetsApi.list(), scenariosApi.list(),
|
||||||
@ -73,24 +78,23 @@ export default function CampaignsPage() {
|
|||||||
setCampaigns(c.data)
|
setCampaigns(c.data)
|
||||||
setTargets(t.data)
|
setTargets(t.data)
|
||||||
setScenarios(s.data)
|
setScenarios(s.data)
|
||||||
// Fetch each campaign's aggregate report so the list can show progress
|
|
||||||
// and overall pass_rate at a glance (activity counts are small).
|
|
||||||
const reports = await Promise.all(
|
|
||||||
c.data.map((camp) =>
|
|
||||||
campaignsApi.report(camp.id).then((r) => r.data).catch(() => null),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
const map: Record<string, CampaignReport> = {}
|
|
||||||
reports.forEach((r) => { if (r) map[r.campaign_id] = r })
|
|
||||||
setReportMap(map)
|
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false)
|
if (!silent) setLoading(false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
useEffect(() => { loadData() }, [])
|
useEffect(() => { loadData() }, [])
|
||||||
useOnTabActive('/campaigns', loadData)
|
useOnTabActive('/campaigns', loadData)
|
||||||
|
|
||||||
|
// Poll the list while this tab is active and a campaign is still working —
|
||||||
|
// compressed dev-line campaigns change fast. Stop once all are terminal.
|
||||||
|
const hasActiveCampaign = campaigns.some((c) => isActiveStatus(c.status))
|
||||||
|
useEffect(() => {
|
||||||
|
if (activeKey !== '/campaigns' || !hasActiveCampaign) return
|
||||||
|
const id = setInterval(() => loadData(true), POLL_INTERVAL_MS)
|
||||||
|
return () => clearInterval(id)
|
||||||
|
}, [activeKey, hasActiveCampaign])
|
||||||
|
|
||||||
const openCreate = () => {
|
const openCreate = () => {
|
||||||
form.setFieldsValue({
|
form.setFieldsValue({
|
||||||
name: '', target_id: undefined, window_seconds: 24 * 3600, time_scale: 1,
|
name: '', target_id: undefined, window_seconds: 24 * 3600, time_scale: 1,
|
||||||
@ -128,50 +132,63 @@ export default function CampaignsPage() {
|
|||||||
loadData()
|
loadData()
|
||||||
}
|
}
|
||||||
|
|
||||||
const openReport = async (campaign: Campaign) => {
|
const fetchReport = async (campaignId: string, silent = false) => {
|
||||||
setReportOpen(true)
|
if (!silent) {
|
||||||
setReportLoading(true)
|
setReportLoading(true)
|
||||||
setReport(null)
|
setReport(null)
|
||||||
setReportRuns([])
|
setReportRuns([])
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
const [rep, runs] = await Promise.all([
|
const [rep, runs] = await Promise.all([
|
||||||
campaignsApi.report(campaign.id),
|
campaignsApi.report(campaignId),
|
||||||
runsApi.list(),
|
runsApi.list(),
|
||||||
])
|
])
|
||||||
setReport(rep.data)
|
setReport(rep.data)
|
||||||
setReportRuns(runs.data.filter((r) => r.campaign_id === campaign.id))
|
setReportRuns(runs.data.filter((r) => r.campaign_id === campaignId))
|
||||||
} finally {
|
} finally {
|
||||||
setReportLoading(false)
|
if (!silent) setReportLoading(false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const openReport = (campaign: CampaignListItem) => {
|
||||||
|
setReportOpen(true)
|
||||||
|
fetchReport(campaign.id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep the open report drawer live while its campaign is still running.
|
||||||
|
const reportId = report?.campaign_id ?? null
|
||||||
|
const reportCampaignActive = campaigns.some(
|
||||||
|
(c) => c.id === reportId && isActiveStatus(c.status),
|
||||||
|
)
|
||||||
|
useEffect(() => {
|
||||||
|
if (activeKey !== '/campaigns' || !reportOpen || !reportId || !reportCampaignActive) return
|
||||||
|
const id = setInterval(() => fetchReport(reportId, true), POLL_INTERVAL_MS)
|
||||||
|
return () => clearInterval(id)
|
||||||
|
}, [activeKey, reportOpen, reportId, reportCampaignActive])
|
||||||
|
|
||||||
const columns = [
|
const columns = [
|
||||||
{ title: '名称', dataIndex: 'name', key: 'name' },
|
{ title: '名称', dataIndex: 'name', key: 'name' },
|
||||||
{ title: '评测对象', key: 'target', render: (_: unknown, c: Campaign) => targetName(c.target_id) },
|
{ title: '评测对象', key: 'target', render: (_: unknown, c: CampaignListItem) => targetName(c.target_id) },
|
||||||
{ title: '窗口', key: 'window', render: (_: unknown, c: Campaign) => fmtWindow(c.window_seconds) },
|
{ title: '窗口', key: 'window', render: (_: unknown, c: CampaignListItem) => fmtWindow(c.window_seconds) },
|
||||||
{
|
{
|
||||||
title: '倍速', key: 'scale',
|
title: '倍速', key: 'scale',
|
||||||
render: (_: unknown, c: Campaign) => (c.time_scale === 1 ? '实时' : `×${c.time_scale}`),
|
render: (_: unknown, c: CampaignListItem) => (c.time_scale === 1 ? '实时' : `×${c.time_scale}`),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: '状态', key: 'status',
|
title: '状态', key: 'status',
|
||||||
render: (_: unknown, c: Campaign) => {
|
render: (_: unknown, c: CampaignListItem) => {
|
||||||
const meta = CAMPAIGN_STATUS[c.status] ?? { label: c.status, color: 'default' }
|
const meta = CAMPAIGN_STATUS[c.status] ?? { label: c.status, color: 'default' }
|
||||||
return <Tag color={meta.color}>{meta.label}</Tag>
|
return <Tag color={meta.color}>{meta.label}</Tag>
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: '进度', key: 'progress',
|
title: '进度', key: 'progress',
|
||||||
render: (_: unknown, c: Campaign) => {
|
render: (_: unknown, c: CampaignListItem) => `${c.progress.completed_runs}/${c.progress.planned_total}`,
|
||||||
const rep = reportMap[c.id]
|
|
||||||
if (!rep) return '—'
|
|
||||||
return `${rep.summary.completed_runs}/${rep.summary.total_runs}`
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: '整窗通过率', key: 'pass_rate',
|
title: '整窗通过率', key: 'pass_rate',
|
||||||
render: (_: unknown, c: Campaign) => {
|
render: (_: unknown, c: CampaignListItem) => {
|
||||||
const rate = reportMap[c.id]?.summary.overall_pass_rate
|
const rate = c.progress.overall_pass_rate
|
||||||
if (rate == null) return '—'
|
if (rate == null) return '—'
|
||||||
return (
|
return (
|
||||||
<Progress
|
<Progress
|
||||||
@ -183,10 +200,10 @@ export default function CampaignsPage() {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: '操作', key: 'actions',
|
title: '操作', key: 'actions',
|
||||||
render: (_: unknown, c: Campaign) => (
|
render: (_: unknown, c: CampaignListItem) => (
|
||||||
<Space>
|
<Space>
|
||||||
<Button size="small" icon={<BarChartOutlined />} onClick={() => openReport(c)}>报告</Button>
|
<Button size="small" icon={<BarChartOutlined />} onClick={() => openReport(c)}>报告</Button>
|
||||||
{(c.status === 'planned' || c.status === 'running') && (
|
{isActiveStatus(c.status) && (
|
||||||
<Popconfirm title="取消该活动?已完成的子运行会保留。" onConfirm={() => cancelCampaign(c.id)}>
|
<Popconfirm title="取消该活动?已完成的子运行会保留。" onConfirm={() => cancelCampaign(c.id)}>
|
||||||
<Button size="small" danger icon={<StopOutlined />}>取消</Button>
|
<Button size="small" danger icon={<StopOutlined />}>取消</Button>
|
||||||
</Popconfirm>
|
</Popconfirm>
|
||||||
@ -243,7 +260,19 @@ export default function CampaignsPage() {
|
|||||||
{ title: '子运行', dataIndex: 'id', key: 'id', render: (id: string) => <code>{id.slice(0, 8)}</code> },
|
{ title: '子运行', dataIndex: 'id', key: 'id', render: (id: string) => <code>{id.slice(0, 8)}</code> },
|
||||||
{
|
{
|
||||||
title: '场景', key: 'scenario',
|
title: '场景', key: 'scenario',
|
||||||
render: (_: unknown, r: Run) => scenarios.find((s) => s.id === r.scenario_id)?.name ?? r.scenario_id.slice(0, 8),
|
render: (_: unknown, r: Run) => (
|
||||||
|
<Space size={4}>
|
||||||
|
<span>{scenarios.find((s) => s.id === r.scenario_id)?.name ?? r.scenario_id.slice(0, 8)}</span>
|
||||||
|
{r.scenario_version != null && <Tag>v{r.scenario_version}</Tag>}
|
||||||
|
</Space>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: '来源', key: 'trigger',
|
||||||
|
render: (_: unknown, r: Run) => {
|
||||||
|
const t = r.triggered_by ?? ''
|
||||||
|
return <Tag color={triggerColors[t] ?? 'default'}>{triggerLabels[t] ?? t}</Tag>
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: '状态', key: 'status',
|
title: '状态', key: 'status',
|
||||||
@ -277,7 +306,7 @@ export default function CampaignsPage() {
|
|||||||
fullHeight
|
fullHeight
|
||||||
extra={
|
extra={
|
||||||
<Space>
|
<Space>
|
||||||
<Button icon={<ReloadOutlined />} onClick={loadData} />
|
<Button icon={<ReloadOutlined />} onClick={() => loadData()} />
|
||||||
<Button type="primary" icon={<PlusOutlined />} onClick={openCreate}>新建活动</Button>
|
<Button type="primary" icon={<PlusOutlined />} onClick={openCreate}>新建活动</Button>
|
||||||
</Space>
|
</Space>
|
||||||
}
|
}
|
||||||
|
|||||||
@ -47,12 +47,14 @@ export const triggerLabels: Record<string, string> = {
|
|||||||
manual: '手动',
|
manual: '手动',
|
||||||
ai_assistant: 'AI 助手',
|
ai_assistant: 'AI 助手',
|
||||||
cli: 'CLI',
|
cli: 'CLI',
|
||||||
|
campaign: '活动',
|
||||||
}
|
}
|
||||||
|
|
||||||
export const triggerColors: Record<string, string> = {
|
export const triggerColors: Record<string, string> = {
|
||||||
manual: 'default',
|
manual: 'default',
|
||||||
ai_assistant: 'purple',
|
ai_assistant: 'purple',
|
||||||
cli: 'blue',
|
cli: 'blue',
|
||||||
|
campaign: 'cyan',
|
||||||
}
|
}
|
||||||
|
|
||||||
export const spacing = {
|
export const spacing = {
|
||||||
|
|||||||
@ -132,6 +132,31 @@ async def test_list_campaigns_after_create(client, seeded_db):
|
|||||||
assert listing[0]["name"] == "24h-cycle"
|
assert listing[0]["name"] == "24h-cycle"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_campaigns_embeds_progress(client, seeded_db):
|
||||||
|
from agenteval.models import EvalRun, RunStatus
|
||||||
|
|
||||||
|
# _valid_payload plan totals 2 + 1 = 3 planned runs.
|
||||||
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||||||
|
repo = RunRepository(seeded_db)
|
||||||
|
repo.create(EvalRun(
|
||||||
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||||||
|
status=RunStatus.COMPLETED,
|
||||||
|
summary={"pass_rate": 1.0, "avg_latency_ms": 100},
|
||||||
|
))
|
||||||
|
repo.create(EvalRun(
|
||||||
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||||||
|
status=RunStatus.COMPLETED,
|
||||||
|
summary={"pass_rate": 0.0, "avg_latency_ms": 200},
|
||||||
|
))
|
||||||
|
|
||||||
|
listing = (await client.get("/api/campaigns")).json()
|
||||||
|
assert len(listing) == 1
|
||||||
|
progress = listing[0]["progress"]
|
||||||
|
assert progress["planned_total"] == 3 # Σ plan.count
|
||||||
|
assert progress["completed_runs"] == 2
|
||||||
|
assert progress["overall_pass_rate"] == 0.5
|
||||||
|
|
||||||
|
|
||||||
async def test_get_campaign_not_found(client, seeded_db):
|
async def test_get_campaign_not_found(client, seeded_db):
|
||||||
resp = await client.get("/api/campaigns/does-not-exist")
|
resp = await client.get("/api/campaigns/does-not-exist")
|
||||||
assert resp.status_code == 404
|
assert resp.status_code == 404
|
||||||
|
|||||||
@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone
|
|||||||
import pytest
|
import pytest
|
||||||
from sqlmodel import Session, SQLModel, create_engine
|
from sqlmodel import Session, SQLModel, create_engine
|
||||||
|
|
||||||
from agenteval.evaluation.report import generate_campaign_report
|
from agenteval.evaluation.report import generate_campaign_report, summarize_campaign_progress
|
||||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus
|
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus
|
||||||
from agenteval.storage.repository import CampaignRepository, RunRepository
|
from agenteval.storage.repository import CampaignRepository, RunRepository
|
||||||
|
|
||||||
@ -151,3 +151,29 @@ def test_time_scale_only_affects_bucketing_not_numbers(report_session):
|
|||||||
assert _bucket(report, 6)["run_count"] == 1
|
assert _bucket(report, 6)["run_count"] == 1
|
||||||
assert _bucket(report, 6)["pass_rate"] == 0.8
|
assert _bucket(report, 6)["pass_rate"] == 0.8
|
||||||
assert _bucket(report, 6)["avg_latency_ms"] == 120.0
|
assert _bucket(report, 6)["avg_latency_ms"] == 120.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_summarize_campaign_progress_uses_planned_total_and_pass_rate(report_session):
|
||||||
|
# plan totals 2 + 1 = 3 planned runs; only two have completed so far.
|
||||||
|
campaign = CampaignRepository(report_session).create(Campaign(
|
||||||
|
name="cycle", target_id="t-1", window_seconds=12, time_scale=1.0,
|
||||||
|
plan=[
|
||||||
|
CampaignPlanEntry(scenario_id="s-a", offset_seconds=0, count=2),
|
||||||
|
CampaignPlanEntry(scenario_id="s-b", offset_seconds=6, count=1),
|
||||||
|
],
|
||||||
|
))
|
||||||
|
campaign.status = CampaignStatus.RUNNING
|
||||||
|
campaign.started_at = T0
|
||||||
|
campaign = CampaignRepository(report_session).update(campaign)
|
||||||
|
|
||||||
|
_seed_child(report_session, campaign.id, "s-a", RunStatus.COMPLETED, 0, pass_rate=1.0)
|
||||||
|
_seed_child(report_session, campaign.id, "s-a", RunStatus.COMPLETED, 1, pass_rate=0.0)
|
||||||
|
_seed_child(report_session, campaign.id, "s-b", RunStatus.FAILED, 6) # execution failure
|
||||||
|
|
||||||
|
runs = RunRepository(report_session).list_by_campaign(campaign.id)
|
||||||
|
progress = summarize_campaign_progress(campaign, runs)
|
||||||
|
|
||||||
|
assert progress["planned_total"] == 3 # Σ plan.count, not spawned-so-far
|
||||||
|
assert progress["completed_runs"] == 2 # FAILED does not count as completed
|
||||||
|
# ADR-0002: failed execution counts as 0.0 → (1.0 + 0.0 + 0.0) / 3
|
||||||
|
assert progress["overall_pass_rate"] == round(1.0 / 3, 4)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user