From c63236f671cea47c770ed9e9e7712ba3b61515d9 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Mon, 3 Aug 2026 01:53:13 +0800 Subject: [PATCH] feat(campaign): support per-campaign analysis model override Campaigns can pin an analysis model config instead of following the global analysis default. Creation validates the referenced config exists (400 otherwise); the create form offers enabled chat configs with the global default as the fallback option. --- backend/agenteval/models.py | 1 + backend/agenteval/storage/db.py | 1 + backend/agenteval/storage/repository.py | 3 ++ backend/agenteval/web/routers/campaigns.py | 7 ++++ frontend/web/src/api.ts | 2 + frontend/web/src/pages/Campaigns.tsx | 35 ++++++++++++++-- ...7e4d3a92c15_add_campaign_analysis_model.py | 29 ++++++++++++++ tests/integration/test_campaigns_api.py | 40 +++++++++++++++++++ .../test_campaign_analysis_model_migration.py | 38 ++++++++++++++++++ 9 files changed, 152 insertions(+), 4 deletions(-) create mode 100644 migrations/versions/b7e4d3a92c15_add_campaign_analysis_model.py create mode 100644 tests/unit/test_campaign_analysis_model_migration.py diff --git a/backend/agenteval/models.py b/backend/agenteval/models.py index 3766cf3..e3302a4 100644 --- a/backend/agenteval/models.py +++ b/backend/agenteval/models.py @@ -289,6 +289,7 @@ class Campaign(BaseModel): completed_at: Optional[datetime] = None created_at: Optional[datetime] = None summary: Optional[CampaignSummary] = None + analysis_model_config_id: Optional[str] = None class Turn(BaseModel): diff --git a/backend/agenteval/storage/db.py b/backend/agenteval/storage/db.py index c36b31f..76e5409 100644 --- a/backend/agenteval/storage/db.py +++ b/backend/agenteval/storage/db.py @@ -187,6 +187,7 @@ class CampaignDB(SQLModel, table=True): completed_at: Optional[datetime] = None created_at: Optional[datetime] = Field(default_factory=utc_now) summary: Optional[str] = None + analysis_model_config_id: Optional[str] = None def get_plan(self) -> list[dict[str, Any]]: return _json_loads(self.plan) diff --git a/backend/agenteval/storage/repository.py b/backend/agenteval/storage/repository.py index 1626965..f3ccfb3 100644 --- a/backend/agenteval/storage/repository.py +++ b/backend/agenteval/storage/repository.py @@ -350,6 +350,7 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]): started_at=campaign.started_at, completed_at=campaign.completed_at, created_at=campaign.created_at, + analysis_model_config_id=campaign.analysis_model_config_id, ) db.set_plan([entry.model_dump(mode="json") for entry in campaign.plan]) if campaign.summary: @@ -369,6 +370,7 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]): completed_at=db.completed_at, created_at=db.created_at, summary=db.get_summary(), + analysis_model_config_id=db.analysis_model_config_id, ) def update(self, campaign: Campaign) -> Optional[Campaign]: @@ -383,6 +385,7 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]): existing.status = campaign.status.value existing.started_at = campaign.started_at existing.completed_at = campaign.completed_at + existing.analysis_model_config_id = campaign.analysis_model_config_id if campaign.summary is not None: existing.set_summary(campaign.summary.model_dump(mode="json")) self.session.add(existing) diff --git a/backend/agenteval/web/routers/campaigns.py b/backend/agenteval/web/routers/campaigns.py index a60bcc0..1fdd390 100644 --- a/backend/agenteval/web/routers/campaigns.py +++ b/backend/agenteval/web/routers/campaigns.py @@ -20,6 +20,7 @@ from agenteval.evaluation.report import ( from agenteval.evaluation.report_render import render_campaign_markdown from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus from agenteval.storage.db import utc_now +from agenteval.storage.model_config_repository import ModelConfigRepository from agenteval.storage.repository import ( CampaignRepository, RunRepository, @@ -37,6 +38,7 @@ class CreateCampaignRequest(BaseModel): window_seconds: int = Field(gt=0) time_scale: float = Field(default=1.0, gt=0) plan: list[CampaignPlanEntry] = Field(min_length=1) + analysis_model_config_id: str | None = None @router.get("") @@ -68,12 +70,17 @@ async def create_campaign( detail=f"scenario not found: {entry.scenario_id}", ) + if request.analysis_model_config_id is not None: + if not ModelConfigRepository(session).get(request.analysis_model_config_id): + raise HTTPException(status_code=400, detail="analysis model config not found") + campaign = Campaign( name=request.name, target_id=request.target_id, window_seconds=request.window_seconds, time_scale=request.time_scale, plan=request.plan, + analysis_model_config_id=request.analysis_model_config_id, ) repo = CampaignRepository(session) campaign = repo.create(campaign) diff --git a/frontend/web/src/api.ts b/frontend/web/src/api.ts index e24cebe..50b9ca3 100644 --- a/frontend/web/src/api.ts +++ b/frontend/web/src/api.ts @@ -336,6 +336,7 @@ export interface Campaign { started_at: string | null completed_at: string | null summary: Record | null + analysis_model_config_id: string | null progress?: CampaignProgress } @@ -399,6 +400,7 @@ export interface CreateCampaignPayload { window_seconds: number time_scale: number plan: CampaignPlanEntry[] + analysis_model_config_id?: string | null } export const campaignsApi = { diff --git a/frontend/web/src/pages/Campaigns.tsx b/frontend/web/src/pages/Campaigns.tsx index 5ccefcb..cad716d 100644 --- a/frontend/web/src/pages/Campaigns.tsx +++ b/frontend/web/src/pages/Campaigns.tsx @@ -13,9 +13,9 @@ import { Line, Bar } from '@ant-design/charts' import PageWrapper from '../components/PageWrapper' import StatCard from '../components/StatCard' import { - campaignsApi, targetsApi, scenariosApi, runsApi, + campaignsApi, targetsApi, scenariosApi, runsApi, modelConfigsApi, type CampaignListItem, type CampaignReport, type Target, type Scenario, type Run, - type CampaignTimelineEntry, + type CampaignTimelineEntry, type ModelConfig, } from '../api' import { passRateColor } from '../utils/colors' import { shortDateTime, toDate } from '../utils/date' @@ -94,6 +94,7 @@ interface CampaignsListsData { campaigns: CampaignListItem[] targets: Target[] scenarios: Scenario[] + modelConfigs: ModelConfig[] } export default function CampaignsPage() { @@ -123,14 +124,18 @@ export default function CampaignsPage() { const { data, loading, reload } = useResource( async () => { - const [c, t, s] = await Promise.all([campaignsApi.list(), targetsApi.list(), scenariosApi.list()]) - return { campaigns: c.data, targets: t.data, scenarios: s.data } + const [c, t, s, m] = await Promise.all([ + campaignsApi.list(), targetsApi.list(), scenariosApi.list(), modelConfigsApi.list(), + ]) + return { campaigns: c.data, targets: t.data, scenarios: s.data, modelConfigs: m.data } }, { tabPath: '/campaigns' }, ) const campaigns = data?.campaigns ?? [] const targets = data?.targets ?? [] const scenarios = data?.scenarios ?? [] + const chatConfigs = (data?.modelConfigs ?? []).filter((m) => m.enabled && m.capability === 'chat') + const analysisDefault = chatConfigs.find((m) => m.is_analysis_default) const scenarioNames = Object.fromEntries(scenarios.map((s) => [s.id, s.name])) const wPlan = (Form.useWatch('plan', form) as PlanFormEntry[] | undefined) ?? [] @@ -163,6 +168,7 @@ export default function CampaignsPage() { name: '', target_id: undefined, window_seconds: 24 * 3600, realtime: false, target_value: 60, target_unit: 60, plan: [{ scenario_id: undefined, offset_hours: 0, count: 1 }], + analysis_model_config_id: null, }) setCreateOpen(true) } @@ -185,6 +191,7 @@ export default function CampaignsPage() { offset_seconds: Math.round((e.offset_hours ?? 0) * 3600), count: e.count ?? 1, })), + analysis_model_config_id: (values.analysis_model_config_id as string | null) ?? null, }) message.success('评估活动已创建并开始调度') setCreateOpen(false) @@ -576,6 +583,26 @@ export default function CampaignsPage() { )} + +