6340ec503c
Resident agents call GET /api/exploration/patrol once per cycle to see every running production-line campaign that opted into exploration (seed set present), the new results since the last watermark (reusing campaign report aggregation), and the remaining exploration budget. The watermark advances after each call so subsequent calls only report increments; accelerated and terminal campaigns are excluded.
441 lines
17 KiB
Python
441 lines
17 KiB
Python
"""Integration tests for the /api/campaigns endpoints.
|
|
|
|
Uses ``httpx.AsyncClient`` with ``app=`` to drive the FastAPI app in-process
|
|
(no real server). No scheduling or child-run spawning happens in this ticket —
|
|
these tests cover persistence, create/query, validation, and that adding the
|
|
nullable ``campaign_id`` column does not disturb existing Run behavior.
|
|
"""
|
|
|
|
import pytest
|
|
from agenteval.models import (
|
|
Case,
|
|
CaseType,
|
|
ChannelType,
|
|
EvalRun,
|
|
EvalTarget,
|
|
PlatformType,
|
|
Scenario,
|
|
TargetStatus,
|
|
)
|
|
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
|
from agenteval.web.app import app
|
|
from httpx import ASGITransport, AsyncClient
|
|
|
|
|
|
@pytest.fixture()
|
|
def seeded_db(db_session, monkeypatch):
|
|
"""Patch the global get_session to the test session and seed target + scenario."""
|
|
from agenteval.storage import db as db_module
|
|
from agenteval.storage import repository as repo_module
|
|
from agenteval.web import app as app_module
|
|
from agenteval.web.routers import campaigns as campaigns_module
|
|
|
|
monkeypatch.setattr(app_module, "init_db", lambda: None)
|
|
# These tests cover persistence/validation/CRUD only — stub out the durable
|
|
# scheduler so creation stays PLANNED and no background loop is launched.
|
|
monkeypatch.setattr(campaigns_module, "start_campaign", lambda *a, **k: None)
|
|
|
|
def _test_get_session():
|
|
return db_session
|
|
|
|
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
|
|
|
from agenteval.web.deps import get_db
|
|
|
|
def _test_get_db():
|
|
try:
|
|
yield db_session
|
|
finally:
|
|
pass
|
|
|
|
app.dependency_overrides[get_db] = _test_get_db
|
|
|
|
target = EvalTarget(
|
|
id="t-1", name="mock-target",
|
|
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={"base_url": "http://mock", "token": "x"},
|
|
status=TargetStatus.ACTIVE,
|
|
)
|
|
TargetRepository(db_session).create(target)
|
|
|
|
scenario = Scenario(
|
|
id="s-1", name="mock-scenario",
|
|
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
|
)
|
|
ScenarioRepository(db_session).create(scenario)
|
|
|
|
yield db_session
|
|
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
@pytest.fixture()
|
|
async def client():
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
def _valid_payload(**overrides) -> dict:
|
|
payload = {
|
|
"name": "24h-cycle",
|
|
"target_id": "t-1",
|
|
"window_seconds": 86400,
|
|
"time_scale": 1.0,
|
|
"plan": [
|
|
{"scenario_id": "s-1", "offset_seconds": 0, "count": 2},
|
|
{"scenario_id": "s-1", "offset_seconds": 3600, "count": 1},
|
|
],
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
# ── list / create / get ──────────────────────────────────────────────────
|
|
|
|
async def test_list_campaigns_empty(client, seeded_db):
|
|
resp = await client.get("/api/campaigns")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
async def test_create_campaign_then_get(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload())
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
campaign_id = body["id"]
|
|
assert body["status"] == "planned"
|
|
assert body["name"] == "24h-cycle"
|
|
assert body["target_id"] == "t-1"
|
|
assert body["window_seconds"] == 86400
|
|
assert body["time_scale"] == 1.0
|
|
assert len(body["plan"]) == 2
|
|
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["id"] == campaign_id
|
|
assert got["status"] == "planned"
|
|
assert got["plan"][0]["scenario_id"] == "s-1"
|
|
assert got["plan"][0]["offset_seconds"] == 0
|
|
assert got["plan"][0]["count"] == 2
|
|
assert got["plan"][1]["offset_seconds"] == 3600
|
|
|
|
|
|
async def test_create_campaign_default_time_scale(client, seeded_db):
|
|
payload = _valid_payload()
|
|
del payload["time_scale"]
|
|
resp = await client.post("/api/campaigns", json=payload)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["time_scale"] == 1.0
|
|
|
|
|
|
async def test_list_campaigns_after_create(client, seeded_db):
|
|
await client.post("/api/campaigns", json=_valid_payload())
|
|
listing = (await client.get("/api/campaigns")).json()
|
|
assert len(listing) == 1
|
|
assert listing[0]["name"] == "24h-cycle"
|
|
|
|
|
|
async def test_list_campaigns_embeds_progress(client, seeded_db):
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
# _valid_payload plan totals 2 + 1 = 3 planned runs.
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
repo = RunRepository(seeded_db)
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"pass_rate": 1.0, "avg_latency_ms": 100},
|
|
))
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"pass_rate": 0.0, "avg_latency_ms": 200},
|
|
))
|
|
|
|
listing = (await client.get("/api/campaigns")).json()
|
|
assert len(listing) == 1
|
|
progress = listing[0]["progress"]
|
|
assert progress["planned_total"] == 3 # Σ plan.count
|
|
assert progress["completed_runs"] == 2
|
|
assert progress["overall_pass_rate"] == 0.5
|
|
|
|
|
|
async def test_get_campaign_not_found(client, seeded_db):
|
|
resp = await client.get("/api/campaigns/does-not-exist")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
# ── validation ─────────────────────────────────────────────────────────────
|
|
|
|
async def test_create_campaign_empty_plan_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(plan=[]))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_create_campaign_missing_target_404(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(target_id="nope"))
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_create_campaign_missing_scenario_404(client, seeded_db):
|
|
payload = _valid_payload(plan=[{"scenario_id": "nope", "offset_seconds": 0, "count": 1}])
|
|
resp = await client.post("/api/campaigns", json=payload)
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_create_campaign_bad_window_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(window_seconds=0))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_create_campaign_bad_time_scale_rejected(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(time_scale=0))
|
|
assert resp.status_code == 422
|
|
|
|
|
|
# ── campaign_id column does not break existing Run behavior ─────────────────
|
|
|
|
async def test_run_campaign_id_defaults_none(seeded_db):
|
|
repo = RunRepository(seeded_db)
|
|
run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1"))
|
|
assert run.campaign_id is None
|
|
fetched = repo.get(run.id)
|
|
assert fetched.campaign_id is None
|
|
|
|
|
|
async def test_run_can_belong_to_campaign(seeded_db):
|
|
repo = RunRepository(seeded_db)
|
|
run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1", campaign_id="camp-1"))
|
|
assert run.campaign_id == "camp-1"
|
|
fetched = repo.get(run.id)
|
|
assert fetched.campaign_id == "camp-1"
|
|
|
|
|
|
# ── cancel endpoint + progress in detail (ticket 03; scheduler stubbed) ─────
|
|
|
|
async def test_cancel_campaign_then_state(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
|
|
resp = await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["status"] == "cancelled"
|
|
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["status"] == "cancelled"
|
|
|
|
|
|
async def test_cancel_already_cancelled_rejected(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
again = await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
|
assert again.status_code == 400
|
|
|
|
|
|
async def test_cancel_missing_campaign_404(client, seeded_db):
|
|
resp = await client.post("/api/campaigns/nope/cancel")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_detail_includes_progress_fields(client, seeded_db):
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert "progress" in got
|
|
progress = got["progress"]
|
|
assert progress["spawned_runs"] == 0
|
|
assert progress["completed_runs"] == 0
|
|
assert progress["current_offset_seconds"] == 0.0
|
|
|
|
|
|
# ── campaign report endpoint (ticket 04) ─────────────────────────────────────
|
|
|
|
async def test_campaign_report_structure_and_values(client, seeded_db):
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
|
|
repo = RunRepository(seeded_db)
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
|
))
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 0, "pass_rate": 0.0, "avg_latency_ms": 200},
|
|
))
|
|
|
|
report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json()
|
|
assert report["campaign_id"] == campaign_id
|
|
assert "time_trend" in report and "capability_summary" in report
|
|
assert report["summary"]["total_runs"] == 2
|
|
assert report["summary"]["completed_runs"] == 2
|
|
assert report["summary"]["overall_pass_rate"] == 0.5
|
|
cap = next(c for c in report["capability_summary"] if c["scenario_id"] == "s-1")
|
|
assert cap["run_count"] == 2
|
|
assert cap["scenario_name"] == "mock-scenario"
|
|
|
|
|
|
async def test_campaign_report_missing_404(client, seeded_db):
|
|
resp = await client.get("/api/campaigns/nope/report")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_campaign_report_markdown_export(client, seeded_db):
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
RunRepository(seeded_db).create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
|
))
|
|
|
|
resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown")
|
|
assert resp.status_code == 200
|
|
assert "text/markdown" in resp.headers["content-type"]
|
|
assert "attachment" in resp.headers["content-disposition"]
|
|
assert "# 活动周期报告" in resp.text
|
|
assert "## 时间趋势" in resp.text
|
|
assert "## 能力汇总" in resp.text
|
|
|
|
|
|
# ── campaign timeline endpoint (ticket 06) ───────────────────────────────────
|
|
|
|
async def test_campaign_timeline_structure(client, seeded_db):
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
|
|
|
repo = RunRepository(seeded_db)
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.COMPLETED,
|
|
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
|
))
|
|
repo.create(EvalRun(
|
|
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
|
status=RunStatus.FAILED,
|
|
))
|
|
|
|
body = (await client.get(f"/api/campaigns/{campaign_id}/timeline")).json()
|
|
assert "entries" in body
|
|
entries = body["entries"]
|
|
assert len(entries) == 2
|
|
for e in entries:
|
|
assert set(e) >= {
|
|
"run_id", "scenario_id", "scenario_name", "offset_seconds",
|
|
"status", "pass_rate", "avg_latency_ms", "started_at",
|
|
}
|
|
assert e["scenario_name"] == "mock-scenario"
|
|
passed = next(e for e in entries if e["status"] == "completed")
|
|
assert passed["pass_rate"] == 1.0
|
|
|
|
|
|
async def test_campaign_timeline_missing_404(client, seeded_db):
|
|
resp = await client.get("/api/campaigns/nope/timeline")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
# ── analysis model override (v0.7 ticket 02) ─────────────────────────────
|
|
|
|
def _seed_chat_config(session, config_id: str = "mc-1") -> None:
|
|
from agenteval.storage.db import ModelConfigDB
|
|
from agenteval.storage.model_config_repository import ModelConfigRepository
|
|
|
|
ModelConfigRepository(session).create(ModelConfigDB(
|
|
id=config_id, name="chat-cfg", provider="openai_compatible", capability="chat",
|
|
endpoint_url="https://models.example.com/v1/chat/completions", model_name="chat-model",
|
|
))
|
|
|
|
|
|
async def test_create_campaign_with_analysis_model_override(client, seeded_db):
|
|
_seed_chat_config(seeded_db)
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(analysis_model_config_id="mc-1"))
|
|
assert resp.status_code == 200
|
|
assert resp.json()["analysis_model_config_id"] == "mc-1"
|
|
|
|
campaign_id = resp.json()["id"]
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["analysis_model_config_id"] == "mc-1"
|
|
listing = (await client.get("/api/campaigns")).json()
|
|
assert listing[0]["analysis_model_config_id"] == "mc-1"
|
|
|
|
|
|
async def test_create_campaign_without_override_stores_null(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload())
|
|
assert resp.status_code == 200
|
|
assert resp.json()["analysis_model_config_id"] is None
|
|
|
|
campaign_id = resp.json()["id"]
|
|
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
|
assert got["analysis_model_config_id"] is None
|
|
|
|
|
|
async def test_create_campaign_invalid_analysis_model_400(client, seeded_db):
|
|
resp = await client.post("/api/campaigns", json=_valid_payload(analysis_model_config_id="nope"))
|
|
assert resp.status_code == 400
|
|
|
|
|
|
async def test_create_campaign_with_exploration_config(client, seeded_db):
|
|
payload = _valid_payload(
|
|
exploration_seeds={"personas": ["急性子用户", "谨慎的老年用户"], "goals": ["查询账单并缴费", "修改收货地址"]},
|
|
exploration_budget={"max_sessions": 4, "max_turns": 6, "min_interval_seconds": 3600},
|
|
)
|
|
resp = await client.post("/api/campaigns", json=payload)
|
|
assert resp.status_code == 200, resp.text
|
|
body = resp.json()
|
|
assert body["exploration_seeds"] == {
|
|
"personas": ["急性子用户", "谨慎的老年用户"],
|
|
"goals": ["查询账单并缴费", "修改收货地址"],
|
|
}
|
|
assert body["exploration_budget"] == {"max_sessions": 4, "max_turns": 6, "min_interval_seconds": 3600}
|
|
|
|
got = (await client.get(f"/api/campaigns/{body['id']}")).json()
|
|
assert got["exploration_seeds"]["personas"] == ["急性子用户", "谨慎的老年用户"]
|
|
assert got["exploration_budget"]["max_turns"] == 6
|
|
|
|
|
|
async def test_create_campaign_without_exploration_config(client, seeded_db):
|
|
body = (await client.post("/api/campaigns", json=_valid_payload())).json()
|
|
assert body["exploration_seeds"] is None
|
|
assert body["exploration_budget"] is None
|
|
|
|
|
|
async def test_empty_seeds_campaign_opts_out_of_exploration(client, seeded_db):
|
|
payload = _valid_payload(exploration_seeds={"personas": [], "goals": []})
|
|
body = (await client.post("/api/campaigns", json=payload)).json()
|
|
assert body["exploration_seeds"] is None
|
|
|
|
|
|
def test_exploration_config_migration_on_existing_db(tmp_path, monkeypatch):
|
|
"""The two campaign config columns apply on a DB at the previous head."""
|
|
from pathlib import Path
|
|
|
|
from agenteval.storage import db as db_module
|
|
from alembic import command
|
|
from alembic.config import Config
|
|
from sqlalchemy import create_engine, inspect, text
|
|
from sqlmodel import SQLModel
|
|
|
|
database_url = f"sqlite:///{tmp_path / 'campaign_config.db'}"
|
|
monkeypatch.setattr(db_module, "DATABASE_URL", database_url)
|
|
config = Config(str(Path(__file__).resolve().parents[2] / "alembic.ini"))
|
|
|
|
SQLModel.metadata.create_all(create_engine(database_url))
|
|
with create_engine(database_url).begin() as connection:
|
|
connection.execute(text("DROP TABLE IF EXISTS exploration_sessions"))
|
|
connection.execute(text("DROP TABLE IF EXISTS exploration_messages"))
|
|
connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_seeds"))
|
|
connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_budget"))
|
|
connection.execute(text("ALTER TABLE campaigns DROP COLUMN last_patrolled_at"))
|
|
connection.execute(text("DROP TABLE IF EXISTS alembic_version"))
|
|
|
|
command.stamp(config, "0e4a7c91d2b3")
|
|
command.upgrade(config, "head")
|
|
|
|
cols = {c["name"] for c in inspect(create_engine(database_url)).get_columns("campaigns")}
|
|
assert {"exploration_seeds", "exploration_budget"} <= cols
|