f433ebb970
Add generate_campaign_report: a pure aggregator over a campaign's child Runs
producing a time-trend axis (Runs bucketed by service-window position) and a
capability-summary axis (grouped by scenario), each carrying pass_rate /
availability / latency. pass_rate keeps the single-Run case-level meaning and
counts execution failures as 0.0 (ADR-0002); time_scale only places Runs into
window-time buckets and never alters any figure. Engine summary now records
avg_latency_ms to feed the latency axis.
Expose GET /api/campaigns/{id}/report (structured) and .../report/markdown
(reusing the existing Markdown export path). Adds "可用性/Availability" to the
domain glossary.
273 lines
10 KiB
Python
273 lines
10 KiB
Python
"""Integration tests for the /api/campaigns endpoints.
|
|
|
|
Uses ``httpx.AsyncClient`` with ``app=`` to drive the FastAPI app in-process
|
|
(no real server). No scheduling or child-run spawning happens in this ticket —
|
|
these tests cover persistence, create/query, validation, and that adding the
|
|
nullable ``campaign_id`` column does not disturb existing Run behavior.
|
|
"""
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
|
|
from agenteval.models import (
|
|
Case, CaseType, ChannelType, EvalRun, EvalTarget, PlatformType,
|
|
Scenario, TargetStatus,
|
|
)
|
|
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
|
from agenteval.web.app import app
|
|
|
|
|
|
@pytest.fixture()
|
|
def seeded_db(db_session, monkeypatch):
|
|
"""Patch the global get_session to the test session and seed target + scenario."""
|
|
from agenteval.storage import db as db_module
|
|
from agenteval.storage import repository as repo_module
|
|
from agenteval.web import app as app_module
|
|
from agenteval.web.routers import campaigns as campaigns_module
|
|
|
|
monkeypatch.setattr(app_module, "init_db", lambda: None)
|
|
# These tests cover persistence/validation/CRUD only — stub out the durable
|
|
# scheduler so creation stays PLANNED and no background loop is launched.
|
|
monkeypatch.setattr(campaigns_module, "start_campaign", lambda *a, **k: None)
|
|
|
|
def _test_get_session():
|
|
return db_session
|
|
|
|
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
|
|
|
from agenteval.web.deps import get_db
|
|
|
|
def _test_get_db():
|
|
try:
|
|
yield db_session
|
|
finally:
|
|
pass
|
|
|
|
app.dependency_overrides[get_db] = _test_get_db
|
|
|
|
target = EvalTarget(
|
|
id="t-1", name="mock-target",
|
|
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={"base_url": "http://mock", "token": "x"},
|
|
status=TargetStatus.ACTIVE,
|
|
)
|
|
TargetRepository(db_session).create(target)
|
|
|
|
scenario = Scenario(
|
|
id="s-1", name="mock-scenario",
|
|
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
|
)
|
|
ScenarioRepository(db_session).create(scenario)
|
|
|
|
yield db_session
|
|
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
@pytest.fixture()
|
|
async def client():
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
def _valid_payload(**overrides) -> dict:
|
|
payload = {
|
|
"name": "24h-cycle",
|
|
"target_id": "t-1",
|
|
"window_seconds": 86400,
|
|
"time_scale": 1.0,
|
|
"plan": [
|
|
{"scenario_id": "s-1", "offset_seconds": 0, "count": 2},
|
|
{"scenario_id": "s-1", "offset_seconds": 3600, "count": 1},
|
|
],
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
# ── list / create / get ──────────────────────────────────────────────────
|
|
|
|
async def test_list_campaigns_empty(client, seeded_db):
|
|
resp = await client.get("/api/campaigns")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
async def test_create_campaign_then_get(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload())
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
campaign_id = body["id"]
|
|
assert body["status"] == "planned"
|
|
assert body["name"] == "24h-cycle"
|
|
assert body["target_id"] == "t-1"
|
|
assert body["window_seconds"] == 86400
|
|
assert body["time_scale"] == 1.0
|
|
assert len(body["plan"]) == 2
|
|
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["id"] == campaign_id
|
|
assert got["status"] == "planned"
|
|
assert got["plan"][0]["scenario_id"] == "s-1"
|
|
assert got["plan"][0]["offset_seconds"] == 0
|
|
assert got["plan"][0]["count"] == 2
|
|
assert got["plan"][1]["offset_seconds"] == 3600
|
|
|
|
|
|
async def test_create_campaign_default_time_scale(client, seeded_db):
|
|
payload = _valid_payload()
|
|
del payload["time_scale"]
|
|
resp = await client.post("/api/campaigns", json=payload)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["time_scale"] == 1.0
|
|
|
|
|
|
async def test_list_campaigns_after_create(client, seeded_db):
|
|
await client.post("/api/campaigns", json=_valid_payload())
|
|
listing = (await client.get("/api/campaigns")).json()
|
|
assert len(listing) == 1
|
|
assert listing[0]["name"] == "24h-cycle"
|
|
|
|
|
|
async def test_get_campaign_not_found(client, seeded_db):
|
|
resp = await client.get("/api/campaigns/does-not-exist")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
# ── validation ─────────────────────────────────────────────────────────────
|
|
|
|
async def test_create_campaign_empty_plan_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(plan=[]))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_create_campaign_missing_target_404(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(target_id="nope"))
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_create_campaign_missing_scenario_404(client, seeded_db):
|
|
payload = _valid_payload(plan=[{"scenario_id": "nope", "offset_seconds": 0, "count": 1}])
|
|
resp = await client.post("/api/campaigns", json=payload)
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_create_campaign_bad_window_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(window_seconds=0))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_create_campaign_bad_time_scale_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(time_scale=0))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
# ── campaign_id column does not break existing Run behavior ─────────────────
|
|
|
|
async def test_run_campaign_id_defaults_none(seeded_db):
|
|
repo = RunRepository(seeded_db)
|
|
run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1"))
|
|
assert run.campaign_id is None
|
|
fetched = repo.get(run.id)
|
|
assert fetched.campaign_id is None
|
|
|
|
|
|
async def test_run_can_belong_to_campaign(seeded_db):
|
|
repo = RunRepository(seeded_db)
|
|
run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1", campaign_id="camp-1"))
|
|
assert run.campaign_id == "camp-1"
|
|
fetched = repo.get(run.id)
|
|
assert fetched.campaign_id == "camp-1"
|
|
|
|
|
|
# ── cancel endpoint + progress in detail (ticket 03; scheduler stubbed) ─────
|
|
|
|
async def test_cancel_campaign_then_state(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
|
|
resp = await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["status"] == "cancelled"
|
|
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["status"] == "cancelled"
|
|
|
|
|
|
async def test_cancel_already_cancelled_rejected(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
again = await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
assert again.status_code == 400
|
|
|
|
|
|
async def test_cancel_missing_campaign_404(client, seeded_db):
|
|
resp = await client.post("/api/campaigns/nope/cancel")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_detail_includes_progress_fields(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert "progress" in got
|
|
progress = got["progress"]
|
|
assert progress["spawned_runs"] == 0
|
|
assert progress["completed_runs"] == 0
|
|
assert progress["current_offset_seconds"] == 0.0
|
|
|
|
|
|
# ── campaign report endpoint (ticket 04) ─────────────────────────────────────
|
|
|
|
async def test_campaign_report_structure_and_values(client, seeded_db):
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
|
|
repo = RunRepository(seeded_db)
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
|
))
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 0, "pass_rate": 0.0, "avg_latency_ms": 200},
|
|
))
|
|
|
|
report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json()
|
|
assert report["campaign_id"] == campaign_id
|
|
assert "time_trend" in report and "capability_summary" in report
|
|
assert report["summary"]["total_runs"] == 2
|
|
assert report["summary"]["completed_runs"] == 2
|
|
assert report["summary"]["overall_pass_rate"] == 0.5
|
|
cap = next(c for c in report["capability_summary"] if c["scenario_id"] == "s-1")
|
|
assert cap["run_count"] == 2
|
|
assert cap["scenario_name"] == "mock-scenario"
|
|
|
|
|
|
async def test_campaign_report_missing_404(client, seeded_db):
|
|
resp = await client.get("/api/campaigns/nope/report")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_campaign_report_markdown_export(client, seeded_db):
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
RunRepository(seeded_db).create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
|
))
|
|
|
|
resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown")
|
|
assert resp.status_code == 200
|
|
assert "text/markdown" in resp.headers["content-type"]
|
|
assert "attachment" in resp.headers["content-disposition"]
|
|
assert "# 活动周期报告" in resp.text
|
|
assert "## 时间趋势" in resp.text
|
|
assert "## 能力汇总" in resp.text
|