diff --git a/backend/agenteval/exploration/models.py b/backend/agenteval/exploration/models.py index 5283d44..2a166f1 100644 --- a/backend/agenteval/exploration/models.py +++ b/backend/agenteval/exploration/models.py @@ -2,8 +2,9 @@ Exploration sessions are an independent entity — never merged into EvalRun — so pass-rate semantics (ADR-0002) and scenario comparability (ADR-0001) stay -untouched. Budget enforcement is a platform ledger: the defaults here are the -hard floor/ceiling, overridable per-campaign (ticket 02 wires the override). +untouched. Budget enforcement is a platform ledger: the defaults here apply +unless overridden per-campaign via ``Campaign.exploration_budget``; the +enforcement itself always happens server-side. """ from datetime import datetime @@ -73,10 +74,17 @@ class ExplorationBudget(BaseModel): def resolve_budget(campaign: Campaign) -> ExplorationBudget: """Effective exploration budget for a campaign. - Currently platform defaults only; the campaign-level override field lands - in ticket 02 and plugs in here. + Platform defaults with per-field campaign overrides; unset override + fields fall back to the defaults. """ - return ExplorationBudget() + override = campaign.exploration_budget + if override is None: + return ExplorationBudget() + return ExplorationBudget( + max_sessions=override.max_sessions or DEFAULT_MAX_SESSIONS_PER_WINDOW, + max_turns=override.max_turns or DEFAULT_MAX_TURNS_PER_SESSION, + min_interval_seconds=override.min_interval_seconds or DEFAULT_MIN_SESSION_INTERVAL_SECONDS, + ) def normalize_experience(raw: dict[str, Any]) -> dict[str, Any]: diff --git a/backend/agenteval/models.py b/backend/agenteval/models.py index e3302a4..9289d84 100644 --- a/backend/agenteval/models.py +++ b/backend/agenteval/models.py @@ -271,6 +271,27 @@ class CampaignSummary(BaseModel): scheduler: SchedulerState = Field(default_factory=SchedulerState) +class ExplorationSeeds(BaseModel): + """Seed set (种子集) for exploratory evaluation: seed personas × seed goals. + + The comparability unit for exploratory evaluation — campaigns sharing a + seed set are comparable across periods. Empty/absent means the campaign + opts out of exploration. + """ + + personas: list[str] = Field(default_factory=list) + goals: list[str] = Field(default_factory=list) + + +class ExplorationBudgetConfig(BaseModel): + """Per-campaign exploration budget override; unset fields fall back to + platform defaults at enforcement time.""" + + max_sessions: Optional[int] = Field(default=None, gt=0) + max_turns: Optional[int] = Field(default=None, gt=0) + min_interval_seconds: Optional[int] = Field(default=None, gt=0) + + class Campaign(BaseModel): """An evaluation campaign: a service-cycle window over a single target, driving many child Runs from a static plan (ADR-0003).""" @@ -290,6 +311,8 @@ class Campaign(BaseModel): created_at: Optional[datetime] = None summary: Optional[CampaignSummary] = None analysis_model_config_id: Optional[str] = None + exploration_seeds: Optional[ExplorationSeeds] = None + exploration_budget: Optional[ExplorationBudgetConfig] = None class Turn(BaseModel): diff --git a/backend/agenteval/storage/db.py b/backend/agenteval/storage/db.py index d627877..4a3c823 100644 --- a/backend/agenteval/storage/db.py +++ b/backend/agenteval/storage/db.py @@ -188,6 +188,8 @@ class CampaignDB(SQLModel, table=True): created_at: Optional[datetime] = Field(default_factory=utc_now) summary: Optional[str] = None analysis_model_config_id: Optional[str] = None + exploration_seeds: Optional[str] = None + exploration_budget: Optional[str] = None def get_plan(self) -> list[dict[str, Any]]: return _json_loads(self.plan) @@ -201,6 +203,18 @@ class CampaignDB(SQLModel, table=True): def set_summary(self, summary: dict[str, Any]) -> None: self.summary = _json_dumps(summary) + def get_exploration_seeds(self) -> Optional[dict[str, Any]]: + return _json_loads(self.exploration_seeds) if self.exploration_seeds else None + + def set_exploration_seeds(self, seeds: dict[str, Any]) -> None: + self.exploration_seeds = _json_dumps(seeds) + + def get_exploration_budget(self) -> Optional[dict[str, Any]]: + return _json_loads(self.exploration_budget) if self.exploration_budget else None + + def set_exploration_budget(self, budget: dict[str, Any]) -> None: + self.exploration_budget = _json_dumps(budget) + class CampaignAnalysisDB(SQLModel, table=True): """One row per campaign holding its intelligent analysis (智能分析) state.""" diff --git a/backend/agenteval/storage/repository.py b/backend/agenteval/storage/repository.py index 43066f1..f64fbfc 100644 --- a/backend/agenteval/storage/repository.py +++ b/backend/agenteval/storage/repository.py @@ -356,6 +356,10 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]): db.set_plan([entry.model_dump(mode="json") for entry in campaign.plan]) if campaign.summary: db.set_summary(campaign.summary.model_dump(mode="json")) + if campaign.exploration_seeds is not None: + db.set_exploration_seeds(campaign.exploration_seeds.model_dump(mode="json")) + if campaign.exploration_budget is not None: + db.set_exploration_budget(campaign.exploration_budget.model_dump(mode="json")) return db def _from_db(self, db: CampaignDB) -> Campaign: @@ -372,6 +376,8 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]): created_at=db.created_at, summary=db.get_summary(), analysis_model_config_id=db.analysis_model_config_id, + exploration_seeds=db.get_exploration_seeds(), + exploration_budget=db.get_exploration_budget(), ) def update(self, campaign: Campaign) -> Optional[Campaign]: @@ -389,6 +395,10 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]): existing.analysis_model_config_id = campaign.analysis_model_config_id if campaign.summary is not None: existing.set_summary(campaign.summary.model_dump(mode="json")) + if campaign.exploration_seeds is not None: + existing.set_exploration_seeds(campaign.exploration_seeds.model_dump(mode="json")) + if campaign.exploration_budget is not None: + existing.set_exploration_budget(campaign.exploration_budget.model_dump(mode="json")) self.session.add(existing) self.session.commit() self.session.refresh(existing) diff --git a/backend/agenteval/web/routers/campaigns.py b/backend/agenteval/web/routers/campaigns.py index 831b73e..6c9c491 100644 --- a/backend/agenteval/web/routers/campaigns.py +++ b/backend/agenteval/web/routers/campaigns.py @@ -26,7 +26,7 @@ from agenteval.evaluation.report import ( summarize_campaign_progress, ) from agenteval.evaluation.report_render import render_campaign_markdown -from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus +from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds from agenteval.storage.db import iso_utc, utc_now from agenteval.storage.model_config_repository import ModelConfigRepository from agenteval.storage.repository import ( @@ -49,6 +49,8 @@ class CreateCampaignRequest(BaseModel): time_scale: float = Field(default=1.0, gt=0) plan: list[CampaignPlanEntry] = Field(min_length=1) analysis_model_config_id: str | None = None + exploration_seeds: ExplorationSeeds | None = None + exploration_budget: ExplorationBudgetConfig | None = None @router.get("") @@ -84,6 +86,10 @@ async def create_campaign( if not ModelConfigRepository(session).get(request.analysis_model_config_id): raise HTTPException(status_code=400, detail="analysis model config not found") + seeds = request.exploration_seeds + if seeds is not None and not seeds.personas and not seeds.goals: + seeds = None # 种子留空 = 该活动不参与探索 + campaign = Campaign( name=request.name, target_id=request.target_id, @@ -91,6 +97,8 @@ async def create_campaign( time_scale=request.time_scale, plan=request.plan, analysis_model_config_id=request.analysis_model_config_id, + exploration_seeds=seeds, + exploration_budget=request.exploration_budget, ) repo = CampaignRepository(session) campaign = repo.create(campaign) diff --git a/frontend/web/src/api.ts b/frontend/web/src/api.ts index 673c131..906e945 100644 --- a/frontend/web/src/api.ts +++ b/frontend/web/src/api.ts @@ -313,6 +313,17 @@ export interface CampaignPlanEntry { count: number } +export interface ExplorationSeeds { + personas: string[] + goals: string[] +} + +export interface ExplorationBudgetConfig { + max_sessions?: number | null + max_turns?: number | null + min_interval_seconds?: number | null +} + export interface CampaignProgress { current_offset_seconds: number spawned_runs: number @@ -337,6 +348,8 @@ export interface Campaign { completed_at: string | null summary: Record | null analysis_model_config_id: string | null + exploration_seeds: ExplorationSeeds | null + exploration_budget: ExplorationBudgetConfig | null progress?: CampaignProgress } @@ -427,6 +440,8 @@ export interface CreateCampaignPayload { time_scale: number plan: CampaignPlanEntry[] analysis_model_config_id?: string | null + exploration_seeds?: ExplorationSeeds | null + exploration_budget?: ExplorationBudgetConfig | null } // ── Period comparison (v0.8) ──────────────────────────────────── diff --git a/frontend/web/src/pages/Campaigns.tsx b/frontend/web/src/pages/Campaigns.tsx index 1e33ab2..0504469 100644 --- a/frontend/web/src/pages/Campaigns.tsx +++ b/frontend/web/src/pages/Campaigns.tsx @@ -83,6 +83,38 @@ function SectionTitle({ children }: { children: ReactNode }) { return
{children}
} +function SeedListText({ name, label, placeholder, addLabel }: { + name: string + label: string + placeholder: string + addLabel: string +}) { + return ( + <> +
{label}
+ + {(fields, { add, remove }) => ( + <> + {fields.map((field) => ( +
+ + + +
+ remove(field.name)} style={{ color: colors.textMuted }} /> +
+
+ ))} + + + )} + + + ) +} + function fmtWindow(seconds: number): string { if (seconds % 3600 === 0) return `${seconds / 3600}h` if (seconds % 60 === 0) return `${seconds / 60}m` @@ -181,6 +213,9 @@ export default function CampaignsPage() { realtime: false, target_value: 60, target_unit: 60, plan: [{ scenario_id: undefined, offset_hours: 0, count: 1 }], analysis_model_config_id: null, + seed_personas: [], seed_goals: [], + exploration_max_sessions: undefined, exploration_max_turns: undefined, + exploration_min_interval_minutes: undefined, }) setCreateOpen(true) } @@ -193,6 +228,16 @@ export default function CampaignsPage() { ) setSubmitting(true) try { + const personas = ((values.seed_personas as string[] | undefined) ?? []) + .map((s) => s?.trim()).filter(Boolean) as string[] + const goals = ((values.seed_goals as string[] | undefined) ?? []) + .map((s) => s?.trim()).filter(Boolean) as string[] + const budget: Record = {} + if (values.exploration_max_sessions != null) budget.max_sessions = values.exploration_max_sessions + if (values.exploration_max_turns != null) budget.max_turns = values.exploration_max_turns + if (values.exploration_min_interval_minutes != null) { + budget.min_interval_seconds = values.exploration_min_interval_minutes * 60 + } await campaignsApi.create({ name: values.name, target_id: values.target_id, @@ -204,6 +249,8 @@ export default function CampaignsPage() { count: e.count ?? 1, })), analysis_model_config_id: (values.analysis_model_config_id as string | null) ?? null, + exploration_seeds: personas.length || goals.length ? { personas, goals } : null, + exploration_budget: Object.keys(budget).length ? budget : null, }) message.success('评估活动已创建并开始调度') setCreateOpen(false) @@ -897,6 +944,35 @@ export default function CampaignsPage() { )} + + + 探索式评测（虚拟用户）{' '} + + + + + + + + + + + + +
+ 探索预算覆盖（留空使用平台默认：8 会话/窗口 · 12 轮/会话 · 30 分钟间隔） +
+
+ + + + + + + + + +
diff --git a/migrations/versions/b3c7d9e1f5a2_add_exploration_config_to_campaigns.py b/migrations/versions/b3c7d9e1f5a2_add_exploration_config_to_campaigns.py new file mode 100644 index 0000000..3ff132d --- /dev/null +++ b/migrations/versions/b3c7d9e1f5a2_add_exploration_config_to_campaigns.py @@ -0,0 +1,29 @@ +"""add exploration seeds/budget config to campaigns + +Revision ID: b3c7d9e1f5a2 +Revises: 0e4a7c91d2b3 +Create Date: 2026-08-03 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +import sqlmodel # noqa: F401 +from alembic import op + +revision: str = "b3c7d9e1f5a2" +down_revision: Union[str, Sequence[str], None] = "0e4a7c91d2b3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table("campaigns") as batch_op: + batch_op.add_column(sa.Column("exploration_seeds", sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + batch_op.add_column(sa.Column("exploration_budget", sqlmodel.sql.sqltypes.AutoString(), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table("campaigns") as batch_op: + batch_op.drop_column("exploration_budget") + batch_op.drop_column("exploration_seeds") diff --git a/tests/integration/test_campaigns_api.py b/tests/integration/test_campaigns_api.py index e002104..58521f2 100644 --- a/tests/integration/test_campaigns_api.py +++ b/tests/integration/test_campaigns_api.py @@ -377,3 +377,63 @@ async def test_create_campaign_without_override_stores_null(client, seeded_db): async def test_create_campaign_invalid_analysis_model_400(client, seeded_db): resp = await client.post("/api/campaigns", json=_valid_payload(analysis_model_config_id="nope")) assert resp.status_code == 400 + + +async def test_create_campaign_with_exploration_config(client, seeded_db): + payload = _valid_payload( + exploration_seeds={"personas": ["急性子用户", "谨慎的老年用户"], "goals": ["查询账单并缴费", "修改收货地址"]}, + exploration_budget={"max_sessions": 4, "max_turns": 6, "min_interval_seconds": 3600}, + ) + resp = await client.post("/api/campaigns", json=payload) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["exploration_seeds"] == { + "personas": ["急性子用户", "谨慎的老年用户"], + "goals": ["查询账单并缴费", "修改收货地址"], + } + assert body["exploration_budget"] == {"max_sessions": 4, "max_turns": 6, "min_interval_seconds": 3600} + + got = (await client.get(f"/api/campaigns/{body['id']}")).json() + assert got["exploration_seeds"]["personas"] == ["急性子用户", "谨慎的老年用户"] + assert got["exploration_budget"]["max_turns"] == 6 + + +async def test_create_campaign_without_exploration_config(client, seeded_db): + body = (await client.post("/api/campaigns", json=_valid_payload())).json() + assert body["exploration_seeds"] is None + assert body["exploration_budget"] is None + + +async def test_empty_seeds_campaign_opts_out_of_exploration(client, seeded_db): + payload = _valid_payload(exploration_seeds={"personas": [], "goals": []}) + body = (await client.post("/api/campaigns", json=payload)).json() + assert body["exploration_seeds"] is None + + +def test_exploration_config_migration_on_existing_db(tmp_path, monkeypatch): + """The two campaign config columns apply on a DB at the previous head.""" + from pathlib import Path + + from agenteval.storage import db as db_module + from alembic import command + from alembic.config import Config + from sqlalchemy import create_engine, inspect, text + from sqlmodel import SQLModel + + database_url = f"sqlite:///{tmp_path / 'campaign_config.db'}" + monkeypatch.setattr(db_module, "DATABASE_URL", database_url) + config = Config(str(Path(__file__).resolve().parents[2] / "alembic.ini")) + + SQLModel.metadata.create_all(create_engine(database_url)) + with create_engine(database_url).begin() as connection: + connection.execute(text("DROP TABLE IF EXISTS exploration_sessions")) + connection.execute(text("DROP TABLE IF EXISTS exploration_messages")) + connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_seeds")) + connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_budget")) + connection.execute(text("DROP TABLE IF EXISTS alembic_version")) + + command.stamp(config, "0e4a7c91d2b3") + command.upgrade(config, "head") + + cols = {c["name"] for c in inspect(create_engine(database_url)).get_columns("campaigns")} + assert {"exploration_seeds", "exploration_budget"} <= cols diff --git a/tests/integration/test_exploration_api.py b/tests/integration/test_exploration_api.py index e79e690..d0184bd 100644 --- a/tests/integration/test_exploration_api.py +++ b/tests/integration/test_exploration_api.py @@ -215,6 +215,19 @@ async def test_session_interval_guardrail(seeded_db, mock_channel, client): assert resp.status_code == 200 +async def test_campaign_budget_override_limits_sessions(seeded_db, mock_channel, client): + """活动级预算覆盖生效（票据 02）：max_sessions=1 时第二个会话被拒。""" + campaign = CampaignRepository(seeded_db).get("c-1") + campaign.exploration_budget = {"max_sessions": 1} + CampaignRepository(seeded_db).update(campaign) + + assert (await _create_session(client)).status_code == 200 + _rewind_latest_session(seeded_db) + resp = await _create_session(client) + assert resp.status_code == 409 + assert "预算" in resp.json()["detail"] + + async def test_turn_budget_guardrail(seeded_db, mock_channel, client): session_id = (await _create_session(client)).json()["id"] for i in range(12): @@ -348,6 +361,8 @@ def test_exploration_migration_on_existing_db(tmp_path, monkeypatch): connection.execute(text("DROP TABLE IF EXISTS exploration_sessions")) connection.execute(text("DROP TABLE IF EXISTS exploration_messages")) + connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_seeds")) + connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_budget")) connection.execute(text("DROP TABLE IF EXISTS alembic_version")) command.stamp(config, "f2a9b7c34d18") diff --git a/tests/unit/test_exploration_budget.py b/tests/unit/test_exploration_budget.py new file mode 100644 index 0000000..7ff3e9d --- /dev/null +++ b/tests/unit/test_exploration_budget.py @@ -0,0 +1,38 @@ +"""Unit tests for exploration budget resolution (platform defaults + campaign override).""" + +from agenteval.exploration.models import resolve_budget +from agenteval.models import Campaign + + +def _campaign(**overrides) -> Campaign: + base = { + "name": "c", + "target_id": "t-1", + "window_seconds": 86400, + "plan": [{"scenario_id": "s-1", "offset_seconds": 0, "count": 1}], + } + base.update(overrides) + return Campaign(**base) + + +def test_platform_defaults_when_no_override(): + budget = resolve_budget(_campaign()) + assert budget.max_sessions == 8 + assert budget.max_turns == 12 + assert budget.min_interval_seconds == 30 * 60 + + +def test_partial_override_falls_back_to_defaults(): + campaign = _campaign(exploration_budget={"max_sessions": 3}) + budget = resolve_budget(campaign) + assert budget.max_sessions == 3 + assert budget.max_turns == 12 + assert budget.min_interval_seconds == 30 * 60 + + +def test_full_override(): + campaign = _campaign( + exploration_budget={"max_sessions": 2, "max_turns": 5, "min_interval_seconds": 600} + ) + budget = resolve_budget(campaign) + assert (budget.max_sessions, budget.max_turns, budget.min_interval_seconds) == (2, 5, 600)