feat(campaigns): exploration seed set and budget config per campaign

v0.9 ticket 02. Campaigns now carry an exploration seed set (seed
personas × seed goals — the comparability unit for exploratory
evaluation) and an optional budget override, stored as JSON columns
isomorphic to plan. Empty seeds normalize to null, marking the campaign
as opted out of exploration. resolve_budget merges per-field overrides
into platform defaults; enforcement stays server-side. The create form
gains seed lists and budget inputs (minutes → seconds), submitting null
when left empty.
This commit is contained in:
sinohqb 2026-08-03 17:52:25 +08:00
parent a351f65550
commit 53afb9b5d1
11 changed files with 302 additions and 6 deletions

View File

@ -2,8 +2,9 @@
Exploration sessions are an independent entity never merged into EvalRun
so pass-rate semantics (ADR-0002) and scenario comparability (ADR-0001) stay
untouched. Budget enforcement is a platform ledger: the defaults here are the
hard floor/ceiling, overridable per-campaign (ticket 02 wires the override).
untouched. Budget enforcement is a platform ledger: the defaults here apply
unless overridden per-campaign via ``Campaign.exploration_budget``; the
enforcement itself always happens server-side.
"""
from datetime import datetime
@ -73,10 +74,17 @@ class ExplorationBudget(BaseModel):
def resolve_budget(campaign: Campaign) -> ExplorationBudget:
"""Effective exploration budget for a campaign.
Currently platform defaults only; the campaign-level override field lands
in ticket 02 and plugs in here.
Platform defaults with per-field campaign overrides; unset override
fields fall back to the defaults.
"""
return ExplorationBudget()
override = campaign.exploration_budget
if override is None:
return ExplorationBudget()
return ExplorationBudget(
max_sessions=override.max_sessions or DEFAULT_MAX_SESSIONS_PER_WINDOW,
max_turns=override.max_turns or DEFAULT_MAX_TURNS_PER_SESSION,
min_interval_seconds=override.min_interval_seconds or DEFAULT_MIN_SESSION_INTERVAL_SECONDS,
)
def normalize_experience(raw: dict[str, Any]) -> dict[str, Any]:

View File

@ -271,6 +271,27 @@ class CampaignSummary(BaseModel):
scheduler: SchedulerState = Field(default_factory=SchedulerState)
class ExplorationSeeds(BaseModel):
"""Seed set (种子集) for exploratory evaluation: seed personas × seed goals.
The comparability unit for exploratory evaluation campaigns sharing a
seed set are comparable across periods. Empty/absent means the campaign
opts out of exploration.
"""
personas: list[str] = Field(default_factory=list)
goals: list[str] = Field(default_factory=list)
class ExplorationBudgetConfig(BaseModel):
"""Per-campaign exploration budget override; unset fields fall back to
platform defaults at enforcement time."""
max_sessions: Optional[int] = Field(default=None, gt=0)
max_turns: Optional[int] = Field(default=None, gt=0)
min_interval_seconds: Optional[int] = Field(default=None, gt=0)
class Campaign(BaseModel):
"""An evaluation campaign: a service-cycle window over a single target,
driving many child Runs from a static plan (ADR-0003)."""
@ -290,6 +311,8 @@ class Campaign(BaseModel):
created_at: Optional[datetime] = None
summary: Optional[CampaignSummary] = None
analysis_model_config_id: Optional[str] = None
exploration_seeds: Optional[ExplorationSeeds] = None
exploration_budget: Optional[ExplorationBudgetConfig] = None
class Turn(BaseModel):

View File

@ -188,6 +188,8 @@ class CampaignDB(SQLModel, table=True):
created_at: Optional[datetime] = Field(default_factory=utc_now)
summary: Optional[str] = None
analysis_model_config_id: Optional[str] = None
exploration_seeds: Optional[str] = None
exploration_budget: Optional[str] = None
def get_plan(self) -> list[dict[str, Any]]:
return _json_loads(self.plan)
@ -201,6 +203,18 @@ class CampaignDB(SQLModel, table=True):
def set_summary(self, summary: dict[str, Any]) -> None:
self.summary = _json_dumps(summary)
def get_exploration_seeds(self) -> Optional[dict[str, Any]]:
return _json_loads(self.exploration_seeds) if self.exploration_seeds else None
def set_exploration_seeds(self, seeds: dict[str, Any]) -> None:
self.exploration_seeds = _json_dumps(seeds)
def get_exploration_budget(self) -> Optional[dict[str, Any]]:
return _json_loads(self.exploration_budget) if self.exploration_budget else None
def set_exploration_budget(self, budget: dict[str, Any]) -> None:
self.exploration_budget = _json_dumps(budget)
class CampaignAnalysisDB(SQLModel, table=True):
"""One row per campaign holding its intelligent analysis (智能分析) state."""

View File

@ -356,6 +356,10 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]):
db.set_plan([entry.model_dump(mode="json") for entry in campaign.plan])
if campaign.summary:
db.set_summary(campaign.summary.model_dump(mode="json"))
if campaign.exploration_seeds is not None:
db.set_exploration_seeds(campaign.exploration_seeds.model_dump(mode="json"))
if campaign.exploration_budget is not None:
db.set_exploration_budget(campaign.exploration_budget.model_dump(mode="json"))
return db
def _from_db(self, db: CampaignDB) -> Campaign:
@ -372,6 +376,8 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]):
created_at=db.created_at,
summary=db.get_summary(),
analysis_model_config_id=db.analysis_model_config_id,
exploration_seeds=db.get_exploration_seeds(),
exploration_budget=db.get_exploration_budget(),
)
def update(self, campaign: Campaign) -> Optional[Campaign]:
@ -389,6 +395,10 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]):
existing.analysis_model_config_id = campaign.analysis_model_config_id
if campaign.summary is not None:
existing.set_summary(campaign.summary.model_dump(mode="json"))
if campaign.exploration_seeds is not None:
existing.set_exploration_seeds(campaign.exploration_seeds.model_dump(mode="json"))
if campaign.exploration_budget is not None:
existing.set_exploration_budget(campaign.exploration_budget.model_dump(mode="json"))
self.session.add(existing)
self.session.commit()
self.session.refresh(existing)

View File

@ -26,7 +26,7 @@ from agenteval.evaluation.report import (
summarize_campaign_progress,
)
from agenteval.evaluation.report_render import render_campaign_markdown
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
from agenteval.storage.db import iso_utc, utc_now
from agenteval.storage.model_config_repository import ModelConfigRepository
from agenteval.storage.repository import (
@ -49,6 +49,8 @@ class CreateCampaignRequest(BaseModel):
time_scale: float = Field(default=1.0, gt=0)
plan: list[CampaignPlanEntry] = Field(min_length=1)
analysis_model_config_id: str | None = None
exploration_seeds: ExplorationSeeds | None = None
exploration_budget: ExplorationBudgetConfig | None = None
@router.get("")
@ -84,6 +86,10 @@ async def create_campaign(
if not ModelConfigRepository(session).get(request.analysis_model_config_id):
raise HTTPException(status_code=400, detail="analysis model config not found")
seeds = request.exploration_seeds
if seeds is not None and not seeds.personas and not seeds.goals:
seeds = None # 种子留空 = 该活动不参与探索
campaign = Campaign(
name=request.name,
target_id=request.target_id,
@ -91,6 +97,8 @@ async def create_campaign(
time_scale=request.time_scale,
plan=request.plan,
analysis_model_config_id=request.analysis_model_config_id,
exploration_seeds=seeds,
exploration_budget=request.exploration_budget,
)
repo = CampaignRepository(session)
campaign = repo.create(campaign)

View File

@ -313,6 +313,17 @@ export interface CampaignPlanEntry {
count: number
}
export interface ExplorationSeeds {
personas: string[]
goals: string[]
}
export interface ExplorationBudgetConfig {
max_sessions?: number | null
max_turns?: number | null
min_interval_seconds?: number | null
}
export interface CampaignProgress {
current_offset_seconds: number
spawned_runs: number
@ -337,6 +348,8 @@ export interface Campaign {
completed_at: string | null
summary: Record<string, unknown> | null
analysis_model_config_id: string | null
exploration_seeds: ExplorationSeeds | null
exploration_budget: ExplorationBudgetConfig | null
progress?: CampaignProgress
}
@ -427,6 +440,8 @@ export interface CreateCampaignPayload {
time_scale: number
plan: CampaignPlanEntry[]
analysis_model_config_id?: string | null
exploration_seeds?: ExplorationSeeds | null
exploration_budget?: ExplorationBudgetConfig | null
}
// ── Period comparison (v0.8) ────────────────────────────────────

View File

@ -83,6 +83,38 @@ function SectionTitle({ children }: { children: ReactNode }) {
return <div style={{ fontWeight: 600, fontSize: 13, margin: '16px 0 12px' }}>{children}</div>
}
function SeedListText({ name, label, placeholder, addLabel }: {
name: string
label: string
placeholder: string
addLabel: string
}) {
return (
<>
<div style={{ marginBottom: 6, fontSize: 12, color: colors.textSecondary }}>{label}</div>
<Form.List name={name}>
{(fields, { add, remove }) => (
<>
{fields.map((field) => (
<div key={field.key} style={{ display: 'flex', gap: 8, marginBottom: 8 }}>
<Form.Item name={field.name} style={{ flex: 1, marginBottom: 0 }}>
<Input placeholder={placeholder} />
</Form.Item>
<div style={{ width: 16, lineHeight: '32px' }}>
<MinusCircleOutlined onClick={() => remove(field.name)} style={{ color: colors.textMuted }} />
</div>
</div>
))}
<Button type="dashed" size="small" onClick={() => add('')} block icon={<PlusOutlined />}>
{addLabel}
</Button>
</>
)}
</Form.List>
</>
)
}
function fmtWindow(seconds: number): string {
if (seconds % 3600 === 0) return `${seconds / 3600}h`
if (seconds % 60 === 0) return `${seconds / 60}m`
@ -181,6 +213,9 @@ export default function CampaignsPage() {
realtime: false, target_value: 60, target_unit: 60,
plan: [{ scenario_id: undefined, offset_hours: 0, count: 1 }],
analysis_model_config_id: null,
seed_personas: [], seed_goals: [],
exploration_max_sessions: undefined, exploration_max_turns: undefined,
exploration_min_interval_minutes: undefined,
})
setCreateOpen(true)
}
@ -193,6 +228,16 @@ export default function CampaignsPage() {
)
setSubmitting(true)
try {
const personas = ((values.seed_personas as string[] | undefined) ?? [])
.map((s) => s?.trim()).filter(Boolean) as string[]
const goals = ((values.seed_goals as string[] | undefined) ?? [])
.map((s) => s?.trim()).filter(Boolean) as string[]
const budget: Record<string, number> = {}
if (values.exploration_max_sessions != null) budget.max_sessions = values.exploration_max_sessions
if (values.exploration_max_turns != null) budget.max_turns = values.exploration_max_turns
if (values.exploration_min_interval_minutes != null) {
budget.min_interval_seconds = values.exploration_min_interval_minutes * 60
}
await campaignsApi.create({
name: values.name,
target_id: values.target_id,
@ -204,6 +249,8 @@ export default function CampaignsPage() {
count: e.count ?? 1,
})),
analysis_model_config_id: (values.analysis_model_config_id as string | null) ?? null,
exploration_seeds: personas.length || goals.length ? { personas, goals } : null,
exploration_budget: Object.keys(budget).length ? budget : null,
})
message.success('评估活动已创建并开始调度')
setCreateOpen(false)
@ -897,6 +944,35 @@ export default function CampaignsPage() {
</>
)}
</Form.List>
<SectionTitle>
{' '}
<Tooltip title="AI 助手以种子人设 × 种子目标自主与被评对象对话;种子留空则该活动不参与探索">
<QuestionCircleOutlined style={{ color: colors.textMuted }} />
</Tooltip>
</SectionTitle>
<Row gutter={16}>
<Col span={12}>
<SeedListText name="seed_personas" label="种子人设" placeholder="如:急性子的缴费用户" addLabel="添加人设" />
</Col>
<Col span={12}>
<SeedListText name="seed_goals" label="种子目标" placeholder="如:查询本月账单并完成缴费" addLabel="添加目标" />
</Col>
</Row>
<div style={{ marginTop: 12, marginBottom: 6, fontSize: 12, color: colors.textSecondary }}>
使8 / · 12 / · 30
</div>
<div style={{ display: 'flex', gap: 8 }}>
<Form.Item name="exploration_max_sessions" style={{ flex: 1, marginBottom: 0 }}>
<InputNumber min={1} placeholder="默认 8" addonAfter="会话/窗口" style={{ width: '100%' }} />
</Form.Item>
<Form.Item name="exploration_max_turns" style={{ flex: 1, marginBottom: 0 }}>
<InputNumber min={1} placeholder="默认 12" addonAfter="轮/会话" style={{ width: '100%' }} />
</Form.Item>
<Form.Item name="exploration_min_interval_minutes" style={{ flex: 1, marginBottom: 0 }}>
<InputNumber min={1} placeholder="默认 30" addonAfter="分钟间隔" style={{ width: '100%' }} />
</Form.Item>
</div>
</Col>
</Row>
</Form>

View File

@ -0,0 +1,29 @@
"""add exploration seeds/budget config to campaigns
Revision ID: b3c7d9e1f5a2
Revises: 0e4a7c91d2b3
Create Date: 2026-08-03
"""
from typing import Sequence, Union
import sqlalchemy as sa
import sqlmodel # noqa: F401
from alembic import op
revision: str = "b3c7d9e1f5a2"
down_revision: Union[str, Sequence[str], None] = "0e4a7c91d2b3"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
with op.batch_alter_table("campaigns") as batch_op:
batch_op.add_column(sa.Column("exploration_seeds", sqlmodel.sql.sqltypes.AutoString(), nullable=True))
batch_op.add_column(sa.Column("exploration_budget", sqlmodel.sql.sqltypes.AutoString(), nullable=True))
def downgrade() -> None:
with op.batch_alter_table("campaigns") as batch_op:
batch_op.drop_column("exploration_budget")
batch_op.drop_column("exploration_seeds")

View File

@ -377,3 +377,63 @@ async def test_create_campaign_without_override_stores_null(client, seeded_db):
async def test_create_campaign_invalid_analysis_model_400(client, seeded_db):
resp = await client.post("/api/campaigns", json=_valid_payload(analysis_model_config_id="nope"))
assert resp.status_code == 400
async def test_create_campaign_with_exploration_config(client, seeded_db):
payload = _valid_payload(
exploration_seeds={"personas": ["急性子用户", "谨慎的老年用户"], "goals": ["查询账单并缴费", "修改收货地址"]},
exploration_budget={"max_sessions": 4, "max_turns": 6, "min_interval_seconds": 3600},
)
resp = await client.post("/api/campaigns", json=payload)
assert resp.status_code == 200, resp.text
body = resp.json()
assert body["exploration_seeds"] == {
"personas": ["急性子用户", "谨慎的老年用户"],
"goals": ["查询账单并缴费", "修改收货地址"],
}
assert body["exploration_budget"] == {"max_sessions": 4, "max_turns": 6, "min_interval_seconds": 3600}
got = (await client.get(f"/api/campaigns/{body['id']}")).json()
assert got["exploration_seeds"]["personas"] == ["急性子用户", "谨慎的老年用户"]
assert got["exploration_budget"]["max_turns"] == 6
async def test_create_campaign_without_exploration_config(client, seeded_db):
body = (await client.post("/api/campaigns", json=_valid_payload())).json()
assert body["exploration_seeds"] is None
assert body["exploration_budget"] is None
async def test_empty_seeds_campaign_opts_out_of_exploration(client, seeded_db):
payload = _valid_payload(exploration_seeds={"personas": [], "goals": []})
body = (await client.post("/api/campaigns", json=payload)).json()
assert body["exploration_seeds"] is None
def test_exploration_config_migration_on_existing_db(tmp_path, monkeypatch):
"""The two campaign config columns apply on a DB at the previous head."""
from pathlib import Path
from agenteval.storage import db as db_module
from alembic import command
from alembic.config import Config
from sqlalchemy import create_engine, inspect, text
from sqlmodel import SQLModel
database_url = f"sqlite:///{tmp_path / 'campaign_config.db'}"
monkeypatch.setattr(db_module, "DATABASE_URL", database_url)
config = Config(str(Path(__file__).resolve().parents[2] / "alembic.ini"))
SQLModel.metadata.create_all(create_engine(database_url))
with create_engine(database_url).begin() as connection:
connection.execute(text("DROP TABLE IF EXISTS exploration_sessions"))
connection.execute(text("DROP TABLE IF EXISTS exploration_messages"))
connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_seeds"))
connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_budget"))
connection.execute(text("DROP TABLE IF EXISTS alembic_version"))
command.stamp(config, "0e4a7c91d2b3")
command.upgrade(config, "head")
cols = {c["name"] for c in inspect(create_engine(database_url)).get_columns("campaigns")}
assert {"exploration_seeds", "exploration_budget"} <= cols

View File

@ -215,6 +215,19 @@ async def test_session_interval_guardrail(seeded_db, mock_channel, client):
assert resp.status_code == 200
async def test_campaign_budget_override_limits_sessions(seeded_db, mock_channel, client):
"""活动级预算覆盖生效(票据 02max_sessions=1 时第二个会话被拒。"""
campaign = CampaignRepository(seeded_db).get("c-1")
campaign.exploration_budget = {"max_sessions": 1}
CampaignRepository(seeded_db).update(campaign)
assert (await _create_session(client)).status_code == 200
_rewind_latest_session(seeded_db)
resp = await _create_session(client)
assert resp.status_code == 409
assert "预算" in resp.json()["detail"]
async def test_turn_budget_guardrail(seeded_db, mock_channel, client):
session_id = (await _create_session(client)).json()["id"]
for i in range(12):
@ -348,6 +361,8 @@ def test_exploration_migration_on_existing_db(tmp_path, monkeypatch):
connection.execute(text("DROP TABLE IF EXISTS exploration_sessions"))
connection.execute(text("DROP TABLE IF EXISTS exploration_messages"))
connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_seeds"))
connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_budget"))
connection.execute(text("DROP TABLE IF EXISTS alembic_version"))
command.stamp(config, "f2a9b7c34d18")

View File

@ -0,0 +1,38 @@
"""Unit tests for exploration budget resolution (platform defaults + campaign override)."""
from agenteval.exploration.models import resolve_budget
from agenteval.models import Campaign
def _campaign(**overrides) -> Campaign:
base = {
"name": "c",
"target_id": "t-1",
"window_seconds": 86400,
"plan": [{"scenario_id": "s-1", "offset_seconds": 0, "count": 1}],
}
base.update(overrides)
return Campaign(**base)
def test_platform_defaults_when_no_override():
budget = resolve_budget(_campaign())
assert budget.max_sessions == 8
assert budget.max_turns == 12
assert budget.min_interval_seconds == 30 * 60
def test_partial_override_falls_back_to_defaults():
campaign = _campaign(exploration_budget={"max_sessions": 3})
budget = resolve_budget(campaign)
assert budget.max_sessions == 3
assert budget.max_turns == 12
assert budget.min_interval_seconds == 30 * 60
def test_full_override():
campaign = _campaign(
exploration_budget={"max_sessions": 2, "max_turns": 5, "min_interval_seconds": 600}
)
budget = resolve_budget(campaign)
assert (budget.max_sessions, budget.max_turns, budget.min_interval_seconds) == (2, 5, 600)