5ecb30876e
- ruff --fix 自动修正 44 项：移除未用 import（pytest 等）、import 块排序归一（I001） - 手工修复剩余 5 项：test_cascade.py 两处未用赋值（F841）；test_s2_rules_and_logic.py 中部 import 移至文件顶部（E402 ×3） - 无行为变更：全量 492 项测试通过
380 lines
14 KiB
Python
380 lines
14 KiB
Python
"""Integration tests for the /api/campaigns endpoints.
|
|
|
|
Uses ``httpx.AsyncClient`` with ``app=`` to drive the FastAPI app in-process
|
|
(no real server). No scheduling or child-run spawning happens in this ticket —
|
|
these tests cover persistence, create/query, validation, and that adding the
|
|
nullable ``campaign_id`` column does not disturb existing Run behavior.
|
|
"""
|
|
|
|
import pytest
|
|
from agenteval.models import (
|
|
Case,
|
|
CaseType,
|
|
ChannelType,
|
|
EvalRun,
|
|
EvalTarget,
|
|
PlatformType,
|
|
Scenario,
|
|
TargetStatus,
|
|
)
|
|
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
|
from agenteval.web.app import app
|
|
from httpx import ASGITransport, AsyncClient
|
|
|
|
|
|
@pytest.fixture()
|
|
def seeded_db(db_session, monkeypatch):
|
|
"""Patch the global get_session to the test session and seed target + scenario."""
|
|
from agenteval.storage import db as db_module
|
|
from agenteval.storage import repository as repo_module
|
|
from agenteval.web import app as app_module
|
|
from agenteval.web.routers import campaigns as campaigns_module
|
|
|
|
monkeypatch.setattr(app_module, "init_db", lambda: None)
|
|
# These tests cover persistence/validation/CRUD only — stub out the durable
|
|
# scheduler so creation stays PLANNED and no background loop is launched.
|
|
monkeypatch.setattr(campaigns_module, "start_campaign", lambda *a, **k: None)
|
|
|
|
def _test_get_session():
|
|
return db_session
|
|
|
|
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
|
|
|
from agenteval.web.deps import get_db
|
|
|
|
def _test_get_db():
|
|
try:
|
|
yield db_session
|
|
finally:
|
|
pass
|
|
|
|
app.dependency_overrides[get_db] = _test_get_db
|
|
|
|
target = EvalTarget(
|
|
id="t-1", name="mock-target",
|
|
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={"base_url": "http://mock", "token": "x"},
|
|
status=TargetStatus.ACTIVE,
|
|
)
|
|
TargetRepository(db_session).create(target)
|
|
|
|
scenario = Scenario(
|
|
id="s-1", name="mock-scenario",
|
|
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
|
)
|
|
ScenarioRepository(db_session).create(scenario)
|
|
|
|
yield db_session
|
|
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
@pytest.fixture()
|
|
async def client():
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
def _valid_payload(**overrides) -> dict:
|
|
payload = {
|
|
"name": "24h-cycle",
|
|
"target_id": "t-1",
|
|
"window_seconds": 86400,
|
|
"time_scale": 1.0,
|
|
"plan": [
|
|
{"scenario_id": "s-1", "offset_seconds": 0, "count": 2},
|
|
{"scenario_id": "s-1", "offset_seconds": 3600, "count": 1},
|
|
],
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
# ── list / create / get ──────────────────────────────────────────────────
|
|
|
|
async def test_list_campaigns_empty(client, seeded_db):
|
|
resp = await client.get("/api/campaigns")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
async def test_create_campaign_then_get(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload())
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
campaign_id = body["id"]
|
|
assert body["status"] == "planned"
|
|
assert body["name"] == "24h-cycle"
|
|
assert body["target_id"] == "t-1"
|
|
assert body["window_seconds"] == 86400
|
|
assert body["time_scale"] == 1.0
|
|
assert len(body["plan"]) == 2
|
|
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["id"] == campaign_id
|
|
assert got["status"] == "planned"
|
|
assert got["plan"][0]["scenario_id"] == "s-1"
|
|
assert got["plan"][0]["offset_seconds"] == 0
|
|
assert got["plan"][0]["count"] == 2
|
|
assert got["plan"][1]["offset_seconds"] == 3600
|
|
|
|
|
|
async def test_create_campaign_default_time_scale(client, seeded_db):
|
|
payload = _valid_payload()
|
|
del payload["time_scale"]
|
|
resp = await client.post("/api/campaigns", json=payload)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["time_scale"] == 1.0
|
|
|
|
|
|
async def test_list_campaigns_after_create(client, seeded_db):
|
|
await client.post("/api/campaigns", json=_valid_payload())
|
|
listing = (await client.get("/api/campaigns")).json()
|
|
assert len(listing) == 1
|
|
assert listing[0]["name"] == "24h-cycle"
|
|
|
|
|
|
async def test_list_campaigns_embeds_progress(client, seeded_db):
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
# _valid_payload plan totals 2 + 1 = 3 planned runs.
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
repo = RunRepository(seeded_db)
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"pass_rate": 1.0, "avg_latency_ms": 100},
|
|
))
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"pass_rate": 0.0, "avg_latency_ms": 200},
|
|
))
|
|
|
|
listing = (await client.get("/api/campaigns")).json()
|
|
assert len(listing) == 1
|
|
progress = listing[0]["progress"]
|
|
assert progress["planned_total"] == 3 # Σ plan.count
|
|
assert progress["completed_runs"] == 2
|
|
assert progress["overall_pass_rate"] == 0.5
|
|
|
|
|
|
async def test_get_campaign_not_found(client, seeded_db):
|
|
resp = await client.get("/api/campaigns/does-not-exist")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
# ── validation ─────────────────────────────────────────────────────────────
|
|
|
|
async def test_create_campaign_empty_plan_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(plan=[]))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_create_campaign_missing_target_404(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(target_id="nope"))
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_create_campaign_missing_scenario_404(client, seeded_db):
|
|
payload = _valid_payload(plan=[{"scenario_id": "nope", "offset_seconds": 0, "count": 1}])
|
|
resp = await client.post("/api/campaigns", json=payload)
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_create_campaign_bad_window_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(window_seconds=0))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_create_campaign_bad_time_scale_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(time_scale=0))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
# ── campaign_id column does not break existing Run behavior ─────────────────
|
|
|
|
async def test_run_campaign_id_defaults_none(seeded_db):
|
|
repo = RunRepository(seeded_db)
|
|
run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1"))
|
|
assert run.campaign_id is None
|
|
fetched = repo.get(run.id)
|
|
assert fetched.campaign_id is None
|
|
|
|
|
|
async def test_run_can_belong_to_campaign(seeded_db):
|
|
repo = RunRepository(seeded_db)
|
|
run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1", campaign_id="camp-1"))
|
|
assert run.campaign_id == "camp-1"
|
|
fetched = repo.get(run.id)
|
|
assert fetched.campaign_id == "camp-1"
|
|
|
|
|
|
# ── cancel endpoint + progress in detail (ticket 03; scheduler stubbed) ─────
|
|
|
|
async def test_cancel_campaign_then_state(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
|
|
resp = await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["status"] == "cancelled"
|
|
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["status"] == "cancelled"
|
|
|
|
|
|
async def test_cancel_already_cancelled_rejected(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
again = await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
assert again.status_code == 400
|
|
|
|
|
|
async def test_cancel_missing_campaign_404(client, seeded_db):
|
|
resp = await client.post("/api/campaigns/nope/cancel")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_detail_includes_progress_fields(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert "progress" in got
|
|
progress = got["progress"]
|
|
assert progress["spawned_runs"] == 0
|
|
assert progress["completed_runs"] == 0
|
|
assert progress["current_offset_seconds"] == 0.0
|
|
|
|
|
|
# ── campaign report endpoint (ticket 04) ─────────────────────────────────────
|
|
|
|
async def test_campaign_report_structure_and_values(client, seeded_db):
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
|
|
repo = RunRepository(seeded_db)
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
|
))
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 0, "pass_rate": 0.0, "avg_latency_ms": 200},
|
|
))
|
|
|
|
report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json()
|
|
assert report["campaign_id"] == campaign_id
|
|
assert "time_trend" in report and "capability_summary" in report
|
|
assert report["summary"]["total_runs"] == 2
|
|
assert report["summary"]["completed_runs"] == 2
|
|
assert report["summary"]["overall_pass_rate"] == 0.5
|
|
cap = next(c for c in report["capability_summary"] if c["scenario_id"] == "s-1")
|
|
assert cap["run_count"] == 2
|
|
assert cap["scenario_name"] == "mock-scenario"
|
|
|
|
|
|
async def test_campaign_report_missing_404(client, seeded_db):
|
|
resp = await client.get("/api/campaigns/nope/report")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_campaign_report_markdown_export(client, seeded_db):
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
RunRepository(seeded_db).create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
|
))
|
|
|
|
resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown")
|
|
assert resp.status_code == 200
|
|
assert "text/markdown" in resp.headers["content-type"]
|
|
assert "attachment" in resp.headers["content-disposition"]
|
|
assert "# 活动周期报告" in resp.text
|
|
assert "## 时间趋势" in resp.text
|
|
assert "## 能力汇总" in resp.text
|
|
|
|
|
|
# ── campaign timeline endpoint (ticket 06) ───────────────────────────────────
|
|
|
|
async def test_campaign_timeline_structure(client, seeded_db):
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
|
|
repo = RunRepository(seeded_db)
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
|
))
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.FAILED,
|
|
))
|
|
|
|
body = (await client.get(f"/api/campaigns/{campaign_id}/timeline")).json()
|
|
assert "entries" in body
|
|
entries = body["entries"]
|
|
assert len(entries) == 2
|
|
for e in entries:
|
|
assert set(e) >= {
|
|
"run_id", "scenario_id", "scenario_name", "offset_seconds",
|
|
"status", "pass_rate", "avg_latency_ms", "started_at",
|
|
}
|
|
assert e["scenario_name"] == "mock-scenario"
|
|
passed = next(e for e in entries if e["status"] == "completed")
|
|
assert passed["pass_rate"] == 1.0
|
|
|
|
|
|
async def test_campaign_timeline_missing_404(client, seeded_db):
|
|
resp = await client.get("/api/campaigns/nope/timeline")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
# ── analysis model override (v0.7 ticket 02) ─────────────────────────────
|
|
|
|
def _seed_chat_config(session, config_id: str = "mc-1") -> None:
|
|
from agenteval.storage.db import ModelConfigDB
|
|
from agenteval.storage.model_config_repository import ModelConfigRepository
|
|
|
|
ModelConfigRepository(session).create(ModelConfigDB(
|
|
id=config_id, name="chat-cfg", provider="openai_compatible", capability="chat",
|
|
endpoint_url="https://models.example.com/v1/chat/completions", model_name="chat-model",
|
|
))
|
|
|
|
|
|
async def test_create_campaign_with_analysis_model_override(client, seeded_db):
|
|
_seed_chat_config(seeded_db)
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(analysis_model_config_id="mc-1"))
|
|
assert resp.status_code == 200
|
|
assert resp.json()["analysis_model_config_id"] == "mc-1"
|
|
|
|
campaign_id = resp.json()["id"]
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["analysis_model_config_id"] == "mc-1"
|
|
listing = (await client.get("/api/campaigns")).json()
|
|
assert listing[0]["analysis_model_config_id"] == "mc-1"
|
|
|
|
|
|
async def test_create_campaign_without_override_stores_null(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload())
|
|
assert resp.status_code == 200
|
|
assert resp.json()["analysis_model_config_id"] is None
|
|
|
|
campaign_id = resp.json()["id"]
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["analysis_model_config_id"] is None
|
|
|
|
|
|
async def test_create_campaign_invalid_analysis_model_400(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(analysis_model_config_id="nope"))
|
|
assert resp.status_code == 400
|