"""Integration tests for the /api/campaigns endpoints. Uses ``httpx.AsyncClient`` with ``app=`` to drive the FastAPI app in-process (no real server). No scheduling or child-run spawning happens in this ticket — these tests cover persistence, create/query, validation, and that adding the nullable ``campaign_id`` column does not disturb existing Run behavior. """ import pytest from agenteval.models import ( Case, CaseType, ChannelType, EvalRun, EvalTarget, PlatformType, Scenario, TargetStatus, ) from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository from agenteval.web.app import app from httpx import ASGITransport, AsyncClient @pytest.fixture() def seeded_db(db_session, monkeypatch): """Patch the global get_session to the test session and seed target + scenario.""" from agenteval.storage import db as db_module from agenteval.storage import repository as repo_module from agenteval.web import app as app_module from agenteval.web.routers import campaigns as campaigns_module monkeypatch.setattr(app_module, "init_db", lambda: None) # These tests cover persistence/validation/CRUD only — stub out the durable # scheduler so creation stays PLANNED and no background loop is launched. monkeypatch.setattr(campaigns_module.campaign_runtime, "start", lambda *a, **k: None) def _test_get_session(): return db_session monkeypatch.setattr(db_module, "get_session", _test_get_session) monkeypatch.setattr(repo_module, "get_session", _test_get_session) from agenteval.web.deps import get_db def _test_get_db(): try: yield db_session finally: pass app.dependency_overrides[get_db] = _test_get_db target = EvalTarget( id="t-1", name="mock-target", platform=PlatformType.AI_DIGITAL_EMPLOYEE, channel_type=ChannelType.TUTU_API, channel_config={"base_url": "http://mock", "token": "x"}, status=TargetStatus.ACTIVE, ) TargetRepository(db_session).create(target) scenario = Scenario( id="s-1", name="mock-scenario", cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])], ) ScenarioRepository(db_session).create(scenario) yield db_session app.dependency_overrides.clear() @pytest.fixture() async def client(): transport = ASGITransport(app=app) async with AsyncClient(transport=transport, base_url="http://test") as c: yield c def _valid_payload(**overrides) -> dict: payload = { "name": "24h-cycle", "target_id": "t-1", "window_seconds": 86400, "time_scale": 1.0, "plan": [ {"scenario_id": "s-1", "offset_seconds": 0, "count": 2}, {"scenario_id": "s-1", "offset_seconds": 3600, "count": 1}, ], } payload.update(overrides) return payload # ── list / create / get ────────────────────────────────────────────────── async def test_list_campaigns_empty(client, seeded_db): resp = await client.get("/api/campaigns") assert resp.status_code == 200 assert resp.json() == [] async def test_create_campaign_then_get(client, seeded_db): resp = await client.post("/api/campaigns", json=_valid_payload()) assert resp.status_code == 200 body = resp.json() campaign_id = body["id"] assert body["status"] == "running" assert body["name"] == "24h-cycle" assert body["target_id"] == "t-1" assert body["window_seconds"] == 86400 assert body["time_scale"] == 1.0 assert len(body["plan"]) == 2 got = (await client.get(f"/api/campaigns/{campaign_id}")).json() assert got["id"] == campaign_id assert got["status"] == "running" assert got["plan"][0]["scenario_id"] == "s-1" assert got["plan"][0]["offset_seconds"] == 0 assert got["plan"][0]["count"] == 2 assert got["plan"][1]["offset_seconds"] == 3600 async def test_create_campaign_default_time_scale(client, seeded_db): payload = _valid_payload() del payload["time_scale"] resp = await client.post("/api/campaigns", json=payload) assert resp.status_code == 200 assert resp.json()["time_scale"] == 1.0 async def test_list_campaigns_after_create(client, seeded_db): await client.post("/api/campaigns", json=_valid_payload()) listing = (await client.get("/api/campaigns")).json() assert len(listing) == 1 assert listing[0]["name"] == "24h-cycle" async def test_list_campaigns_embeds_progress(client, seeded_db): from agenteval.models import EvalRun, RunStatus # _valid_payload plan totals 2 + 1 = 3 planned runs. campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] repo = RunRepository(seeded_db) repo.create(EvalRun( target_id="t-1", scenario_id="s-1", campaign_id=campaign_id, status=RunStatus.COMPLETED, summary={"pass_rate": 1.0, "avg_latency_ms": 100}, )) repo.create(EvalRun( target_id="t-1", scenario_id="s-1", campaign_id=campaign_id, status=RunStatus.COMPLETED, summary={"pass_rate": 0.0, "avg_latency_ms": 200}, )) listing = (await client.get("/api/campaigns")).json() assert len(listing) == 1 progress = listing[0]["progress"] assert progress["planned_total"] == 3 # Σ plan.count assert progress["completed_runs"] == 2 assert progress["overall_pass_rate"] == 0.5 async def test_get_campaign_not_found(client, seeded_db): resp = await client.get("/api/campaigns/does-not-exist") assert resp.status_code == 404 # ── validation ───────────────────────────────────────────────────────────── async def test_create_campaign_empty_plan_rejected(client, seeded_db): resp = await client.post("/api/campaigns", json=_valid_payload(plan=[])) assert resp.status_code == 422 async def test_create_campaign_missing_target_404(client, seeded_db): resp = await client.post("/api/campaigns", json=_valid_payload(target_id="nope")) assert resp.status_code == 404 async def test_create_campaign_missing_scenario_404(client, seeded_db): payload = _valid_payload(plan=[{"scenario_id": "nope", "offset_seconds": 0, "count": 1}]) resp = await client.post("/api/campaigns", json=payload) assert resp.status_code == 404 async def test_create_campaign_bad_window_rejected(client, seeded_db): resp = await client.post("/api/campaigns", json=_valid_payload(window_seconds=0)) assert resp.status_code == 422 async def test_create_campaign_bad_time_scale_rejected(client, seeded_db): resp = await client.post("/api/campaigns", json=_valid_payload(time_scale=0)) assert resp.status_code == 422 # ── campaign_id column does not break existing Run behavior ───────────────── async def test_run_campaign_id_defaults_none(seeded_db): repo = RunRepository(seeded_db) run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1")) assert run.campaign_id is None fetched = repo.get(run.id) assert fetched.campaign_id is None async def test_run_can_belong_to_campaign(seeded_db): repo = RunRepository(seeded_db) run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1", campaign_id="camp-1")) assert run.campaign_id == "camp-1" fetched = repo.get(run.id) assert fetched.campaign_id == "camp-1" # ── cancel endpoint + progress in detail (ticket 03; scheduler stubbed) ───── async def test_cancel_campaign_then_state(client, seeded_db): campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] resp = await client.post(f"/api/campaigns/{campaign_id}/cancel") assert resp.status_code == 200 assert resp.json()["status"] == "cancelled" got = (await client.get(f"/api/campaigns/{campaign_id}")).json() assert got["status"] == "cancelled" async def test_cancel_already_cancelled_rejected(client, seeded_db): campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] await client.post(f"/api/campaigns/{campaign_id}/cancel") again = await client.post(f"/api/campaigns/{campaign_id}/cancel") assert again.status_code == 400 async def test_cancel_missing_campaign_404(client, seeded_db): resp = await client.post("/api/campaigns/nope/cancel") assert resp.status_code == 404 async def test_detail_includes_progress_fields(client, seeded_db): campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] got = (await client.get(f"/api/campaigns/{campaign_id}")).json() assert "progress" in got progress = got["progress"] assert progress["spawned_runs"] == 0 assert progress["completed_runs"] == 0 assert progress["current_offset_seconds"] >= 0.0 # ── campaign report endpoint (ticket 04) ───────────────────────────────────── async def test_campaign_report_structure_and_values(client, seeded_db): from agenteval.models import EvalRun, RunStatus campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] repo = RunRepository(seeded_db) repo.create(EvalRun( target_id="t-1", scenario_id="s-1", campaign_id=campaign_id, status=RunStatus.COMPLETED, summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100}, )) repo.create(EvalRun( target_id="t-1", scenario_id="s-1", campaign_id=campaign_id, status=RunStatus.COMPLETED, summary={"total_cases": 1, "passed_cases": 0, "pass_rate": 0.0, "avg_latency_ms": 200}, )) report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json() assert report["campaign_id"] == campaign_id assert "time_trend" in report and "capability_summary" in report assert report["summary"]["total_runs"] == 2 assert report["summary"]["completed_runs"] == 2 assert report["summary"]["overall_pass_rate"] == 0.5 cap = next(c for c in report["capability_summary"] if c["scenario_id"] == "s-1") assert cap["run_count"] == 2 assert cap["scenario_name"] == "mock-scenario" async def test_campaign_report_missing_404(client, seeded_db): resp = await client.get("/api/campaigns/nope/report") assert resp.status_code == 404 async def test_campaign_report_markdown_export(client, seeded_db): from agenteval.models import EvalRun, RunStatus campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] RunRepository(seeded_db).create(EvalRun( target_id="t-1", scenario_id="s-1", campaign_id=campaign_id, status=RunStatus.COMPLETED, summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100}, )) resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown") assert resp.status_code == 200 assert "text/markdown" in resp.headers["content-type"] assert "attachment" in resp.headers["content-disposition"] assert "# 活动周期报告" in resp.text assert "## 时间趋势" in resp.text assert "## 能力汇总" in resp.text # ── exploration findings in report/export (v0.9 ticket 05) ─────────────────── def _seed_exploration_session(seeded_db, campaign_id: str) -> None: from agenteval.exploration.models import ExplorationSession from agenteval.storage.repository import ExplorationSessionRepository repo = ExplorationSessionRepository(seeded_db) session_obj = repo.create( ExplorationSession(campaign_id=campaign_id, target_id="t-1", persona={"name": "x"}, goal="缴费") ) session_obj.experience = { "goal_achieved": True, "blockers": ["缴费入口难找"], "misled": [], "emotion": "neutral", "notes": "", } repo.update(session_obj) async def test_campaign_report_includes_exploration_findings(client, seeded_db): campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] _seed_exploration_session(seeded_db, campaign_id) report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json() exploration = report["exploration"] assert exploration["session_count"] == 1 assert exploration["goal_achievement_rate"] == 1.0 assert exploration["issues"] == [{"issue": "缴费入口难找", "count": 1}] async def test_campaign_report_without_exploration_omits_key(client, seeded_db): campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json() assert "exploration" not in report async def test_markdown_export_appends_exploration_appendix(client, seeded_db): campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] _seed_exploration_session(seeded_db, campaign_id) resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown") assert resp.status_code == 200 assert "## 探索发现" in resp.text assert "缴费入口难找 ×1" in resp.text async def test_markdown_export_without_exploration_leaves_no_trace(client, seeded_db): campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown") assert resp.status_code == 200 assert "探索发现" not in resp.text # ── campaign timeline endpoint (ticket 06) ─────────────────────────────────── async def test_campaign_timeline_structure(client, seeded_db): from agenteval.models import EvalRun, RunStatus campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"] repo = RunRepository(seeded_db) repo.create(EvalRun( target_id="t-1", scenario_id="s-1", campaign_id=campaign_id, status=RunStatus.COMPLETED, summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100}, )) repo.create(EvalRun( target_id="t-1", scenario_id="s-1", campaign_id=campaign_id, status=RunStatus.FAILED, )) body = (await client.get(f"/api/campaigns/{campaign_id}/timeline")).json() assert "entries" in body entries = body["entries"] assert len(entries) == 2 for e in entries: assert set(e) >= { "run_id", "scenario_id", "scenario_name", "offset_seconds", "status", "pass_rate", "avg_latency_ms", "started_at", } assert e["scenario_name"] == "mock-scenario" passed = next(e for e in entries if e["status"] == "completed") assert passed["pass_rate"] == 1.0 async def test_campaign_timeline_missing_404(client, seeded_db): resp = await client.get("/api/campaigns/nope/timeline") assert resp.status_code == 404 # ── analysis model override (v0.7 ticket 02) ───────────────────────────── def _seed_chat_config(session, config_id: str = "mc-1") -> None: from agenteval.storage.db import ModelConfigDB from agenteval.storage.model_config_repository import ModelConfigRepository ModelConfigRepository(session).create(ModelConfigDB( id=config_id, name="chat-cfg", provider="openai_compatible", capability="chat", endpoint_url="https://models.example.com/v1/chat/completions", model_name="chat-model", )) async def test_create_campaign_with_analysis_model_override(client, seeded_db): _seed_chat_config(seeded_db) resp = await client.post("/api/campaigns", json=_valid_payload(analysis_model_config_id="mc-1")) assert resp.status_code == 200 assert resp.json()["analysis_model_config_id"] == "mc-1" campaign_id = resp.json()["id"] got = (await client.get(f"/api/campaigns/{campaign_id}")).json() assert got["analysis_model_config_id"] == "mc-1" listing = (await client.get("/api/campaigns")).json() assert listing[0]["analysis_model_config_id"] == "mc-1" async def test_create_campaign_without_override_stores_null(client, seeded_db): resp = await client.post("/api/campaigns", json=_valid_payload()) assert resp.status_code == 200 assert resp.json()["analysis_model_config_id"] is None campaign_id = resp.json()["id"] got = (await client.get(f"/api/campaigns/{campaign_id}")).json() assert got["analysis_model_config_id"] is None async def test_create_campaign_invalid_analysis_model_400(client, seeded_db): resp = await client.post("/api/campaigns", json=_valid_payload(analysis_model_config_id="nope")) assert resp.status_code == 400 async def test_create_campaign_with_exploration_config(client, seeded_db): payload = _valid_payload( exploration_seeds={"personas": ["急性子用户", "谨慎的老年用户"], "goals": ["查询账单并缴费", "修改收货地址"]}, exploration_budget={"max_sessions": 4, "max_turns": 6, "min_interval_seconds": 3600}, ) resp = await client.post("/api/campaigns", json=payload) assert resp.status_code == 200, resp.text body = resp.json() assert body["exploration_seeds"] == { "personas": ["急性子用户", "谨慎的老年用户"], "goals": ["查询账单并缴费", "修改收货地址"], } assert body["exploration_budget"] == {"max_sessions": 4, "max_turns": 6, "min_interval_seconds": 3600} got = (await client.get(f"/api/campaigns/{body['id']}")).json() assert got["exploration_seeds"]["personas"] == ["急性子用户", "谨慎的老年用户"] assert got["exploration_budget"]["max_turns"] == 6 async def test_create_campaign_without_exploration_config(client, seeded_db): body = (await client.post("/api/campaigns", json=_valid_payload())).json() assert body["exploration_seeds"] is None assert body["exploration_budget"] is None async def test_empty_seeds_campaign_opts_out_of_exploration(client, seeded_db): payload = _valid_payload(exploration_seeds={"personas": [], "goals": []}) body = (await client.post("/api/campaigns", json=payload)).json() assert body["exploration_seeds"] is None def test_exploration_config_migration_on_existing_db(tmp_path, monkeypatch): """The two campaign config columns apply on a DB at the previous head.""" from pathlib import Path from agenteval.storage import db as db_module from alembic import command from alembic.config import Config from sqlalchemy import create_engine, inspect, text from sqlmodel import SQLModel database_url = f"sqlite:///{tmp_path / 'campaign_config.db'}" monkeypatch.setattr(db_module, "DATABASE_URL", database_url) config = Config(str(Path(__file__).resolve().parents[2] / "alembic.ini")) SQLModel.metadata.create_all(create_engine(database_url)) with create_engine(database_url).begin() as connection: connection.execute(text("DROP TABLE IF EXISTS intelligent_eval_messages")) connection.execute(text("DROP TABLE IF EXISTS intelligent_eval_sessions")) connection.execute(text("DROP TABLE IF EXISTS intelligent_evals")) connection.execute(text("DROP TABLE IF EXISTS exploration_sessions")) connection.execute(text("DROP TABLE IF EXISTS exploration_messages")) connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_seeds")) connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_budget")) connection.execute(text("ALTER TABLE campaigns DROP COLUMN last_patrolled_at")) connection.execute(text("DROP TABLE IF EXISTS alembic_version")) command.stamp(config, "0e4a7c91d2b3") command.upgrade(config, "head") cols = {c["name"] for c in inspect(create_engine(database_url)).get_columns("campaigns")} assert {"exploration_seeds", "exploration_budget"} <= cols