8910fd17e0
Add a thin async loop (run_campaign_loop) that ticks on real wall-clock time,
maps elapsed×time_scale to a window offset via the pure decide_schedule, spawns
due child Runs, and marks the campaign COMPLETED at window end. All authority
lives in the DB (started_at, spawned_indices, status), so the app lifespan can
resume every RUNNING campaign on startup without double-spawning and stop all
loops gracefully on shutdown. A failing plan entry is skipped and recorded
rather than wedging the campaign.
Creating a campaign now starts its loop; POST /api/campaigns/{id}/cancel stops
further spawning (completed child Runs are kept); GET /api/campaigns/{id}
reports live progress (window offset, spawned/completed Run counts).
219 lines
7.9 KiB
Python
219 lines
7.9 KiB
Python
"""Integration tests for the /api/campaigns endpoints.
|
|
|
|
Uses ``httpx.AsyncClient`` with ``app=`` to drive the FastAPI app in-process
|
|
(no real server). No scheduling or child-run spawning happens in this ticket —
|
|
these tests cover persistence, create/query, validation, and that adding the
|
|
nullable ``campaign_id`` column does not disturb existing Run behavior.
|
|
"""
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
|
|
from agenteval.models import (
|
|
Case, CaseType, ChannelType, EvalRun, EvalTarget, PlatformType,
|
|
Scenario, TargetStatus,
|
|
)
|
|
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
|
from agenteval.web.app import app
|
|
|
|
|
|
@pytest.fixture()
|
|
def seeded_db(db_session, monkeypatch):
|
|
"""Patch the global get_session to the test session and seed target + scenario."""
|
|
from agenteval.storage import db as db_module
|
|
from agenteval.storage import repository as repo_module
|
|
from agenteval.web import app as app_module
|
|
from agenteval.web.routers import campaigns as campaigns_module
|
|
|
|
monkeypatch.setattr(app_module, "init_db", lambda: None)
|
|
# These tests cover persistence/validation/CRUD only — stub out the durable
|
|
# scheduler so creation stays PLANNED and no background loop is launched.
|
|
monkeypatch.setattr(campaigns_module, "start_campaign", lambda *a, **k: None)
|
|
|
|
def _test_get_session():
|
|
return db_session
|
|
|
|
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
|
|
|
from agenteval.web.deps import get_db
|
|
|
|
def _test_get_db():
|
|
try:
|
|
yield db_session
|
|
finally:
|
|
pass
|
|
|
|
app.dependency_overrides[get_db] = _test_get_db
|
|
|
|
target = EvalTarget(
|
|
id="t-1", name="mock-target",
|
|
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={"base_url": "http://mock", "token": "x"},
|
|
status=TargetStatus.ACTIVE,
|
|
)
|
|
TargetRepository(db_session).create(target)
|
|
|
|
scenario = Scenario(
|
|
id="s-1", name="mock-scenario",
|
|
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
|
)
|
|
ScenarioRepository(db_session).create(scenario)
|
|
|
|
yield db_session
|
|
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
@pytest.fixture()
|
|
async def client():
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
def _valid_payload(**overrides) -> dict:
|
|
payload = {
|
|
"name": "24h-cycle",
|
|
"target_id": "t-1",
|
|
"window_seconds": 86400,
|
|
"time_scale": 1.0,
|
|
"plan": [
|
|
{"scenario_id": "s-1", "offset_seconds": 0, "count": 2},
|
|
{"scenario_id": "s-1", "offset_seconds": 3600, "count": 1},
|
|
],
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
# ── list / create / get ──────────────────────────────────────────────────
|
|
|
|
async def test_list_campaigns_empty(client, seeded_db):
|
|
resp = await client.get("/api/campaigns")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
async def test_create_campaign_then_get(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload())
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
campaign_id = body["id"]
|
|
assert body["status"] == "planned"
|
|
assert body["name"] == "24h-cycle"
|
|
assert body["target_id"] == "t-1"
|
|
assert body["window_seconds"] == 86400
|
|
assert body["time_scale"] == 1.0
|
|
assert len(body["plan"]) == 2
|
|
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["id"] == campaign_id
|
|
assert got["status"] == "planned"
|
|
assert got["plan"][0]["scenario_id"] == "s-1"
|
|
assert got["plan"][0]["offset_seconds"] == 0
|
|
assert got["plan"][0]["count"] == 2
|
|
assert got["plan"][1]["offset_seconds"] == 3600
|
|
|
|
|
|
async def test_create_campaign_default_time_scale(client, seeded_db):
|
|
payload = _valid_payload()
|
|
del payload["time_scale"]
|
|
resp = await client.post("/api/campaigns", json=payload)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["time_scale"] == 1.0
|
|
|
|
|
|
async def test_list_campaigns_after_create(client, seeded_db):
|
|
await client.post("/api/campaigns", json=_valid_payload())
|
|
listing = (await client.get("/api/campaigns")).json()
|
|
assert len(listing) == 1
|
|
assert listing[0]["name"] == "24h-cycle"
|
|
|
|
|
|
async def test_get_campaign_not_found(client, seeded_db):
|
|
resp = await client.get("/api/campaigns/does-not-exist")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
# ── validation ─────────────────────────────────────────────────────────────
|
|
|
|
async def test_create_campaign_empty_plan_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(plan=[]))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_create_campaign_missing_target_404(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(target_id="nope"))
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_create_campaign_missing_scenario_404(client, seeded_db):
|
|
payload = _valid_payload(plan=[{"scenario_id": "nope", "offset_seconds": 0, "count": 1}])
|
|
resp = await client.post("/api/campaigns", json=payload)
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_create_campaign_bad_window_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(window_seconds=0))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_create_campaign_bad_time_scale_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(time_scale=0))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
# ── campaign_id column does not break existing Run behavior ─────────────────
|
|
|
|
async def test_run_campaign_id_defaults_none(seeded_db):
|
|
repo = RunRepository(seeded_db)
|
|
run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1"))
|
|
assert run.campaign_id is None
|
|
fetched = repo.get(run.id)
|
|
assert fetched.campaign_id is None
|
|
|
|
|
|
async def test_run_can_belong_to_campaign(seeded_db):
|
|
repo = RunRepository(seeded_db)
|
|
run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1", campaign_id="camp-1"))
|
|
assert run.campaign_id == "camp-1"
|
|
fetched = repo.get(run.id)
|
|
assert fetched.campaign_id == "camp-1"
|
|
|
|
|
|
# ── cancel endpoint + progress in detail (ticket 03; scheduler stubbed) ─────
|
|
|
|
async def test_cancel_campaign_then_state(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
|
|
resp = await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["status"] == "cancelled"
|
|
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["status"] == "cancelled"
|
|
|
|
|
|
async def test_cancel_already_cancelled_rejected(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
again = await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
assert again.status_code == 400
|
|
|
|
|
|
async def test_cancel_missing_campaign_404(client, seeded_db):
|
|
resp = await client.post("/api/campaigns/nope/cancel")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_detail_includes_progress_fields(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert "progress" in got
|
|
progress = got["progress"]
|
|
assert progress["spawned_runs"] == 0
|
|
assert progress["completed_runs"] == 0
|
|
assert progress["current_offset_seconds"] == 0.0
|