1317552701
Introduce 智能评估 as an evaluation paradigm parallel to static evaluation, driven by OpenClaw. The platform supplies storage, lifecycle, and reporting; OpenClaw plans and executes. - Data model: IntelligentEval + Session + Message tables (new, not reusing exploration) - Lifecycle state machine: draft → planning → pending_approval → executing → completed/cancelled/failed - Session API: create/message (channel-forwarded)/close with turn accounting - Report API: pydantic-validated structured report, executing → completed, Markdown export (pure renderer) - Alembic migration for the three tables; domain glossary added to CONTEXT.md
499 lines
20 KiB
Python
499 lines
20 KiB
Python
"""Integration tests for the /api/campaigns endpoints.
|
||
|
||
Uses ``httpx.AsyncClient`` with ``app=`` to drive the FastAPI app in-process
|
||
(no real server). No scheduling or child-run spawning happens in this ticket —
|
||
these tests cover persistence, create/query, validation, and that adding the
|
||
nullable ``campaign_id`` column does not disturb existing Run behavior.
|
||
"""
|
||
|
||
import pytest
|
||
from agenteval.models import (
|
||
Case,
|
||
CaseType,
|
||
ChannelType,
|
||
EvalRun,
|
||
EvalTarget,
|
||
PlatformType,
|
||
Scenario,
|
||
TargetStatus,
|
||
)
|
||
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
||
from agenteval.web.app import app
|
||
from httpx import ASGITransport, AsyncClient
|
||
|
||
|
||
@pytest.fixture()
|
||
def seeded_db(db_session, monkeypatch):
|
||
"""Patch the global get_session to the test session and seed target + scenario."""
|
||
from agenteval.storage import db as db_module
|
||
from agenteval.storage import repository as repo_module
|
||
from agenteval.web import app as app_module
|
||
from agenteval.web.routers import campaigns as campaigns_module
|
||
|
||
monkeypatch.setattr(app_module, "init_db", lambda: None)
|
||
# These tests cover persistence/validation/CRUD only — stub out the durable
|
||
# scheduler so creation stays PLANNED and no background loop is launched.
|
||
monkeypatch.setattr(campaigns_module, "start_campaign", lambda *a, **k: None)
|
||
|
||
def _test_get_session():
|
||
return db_session
|
||
|
||
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
||
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
||
|
||
from agenteval.web.deps import get_db
|
||
|
||
def _test_get_db():
|
||
try:
|
||
yield db_session
|
||
finally:
|
||
pass
|
||
|
||
app.dependency_overrides[get_db] = _test_get_db
|
||
|
||
target = EvalTarget(
|
||
id="t-1", name="mock-target",
|
||
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
||
channel_type=ChannelType.TUTU_API,
|
||
channel_config={"base_url": "http://mock", "token": "x"},
|
||
status=TargetStatus.ACTIVE,
|
||
)
|
||
TargetRepository(db_session).create(target)
|
||
|
||
scenario = Scenario(
|
||
id="s-1", name="mock-scenario",
|
||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||
)
|
||
ScenarioRepository(db_session).create(scenario)
|
||
|
||
yield db_session
|
||
|
||
app.dependency_overrides.clear()
|
||
|
||
|
||
@pytest.fixture()
|
||
async def client():
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
def _valid_payload(**overrides) -> dict:
|
||
payload = {
|
||
"name": "24h-cycle",
|
||
"target_id": "t-1",
|
||
"window_seconds": 86400,
|
||
"time_scale": 1.0,
|
||
"plan": [
|
||
{"scenario_id": "s-1", "offset_seconds": 0, "count": 2},
|
||
{"scenario_id": "s-1", "offset_seconds": 3600, "count": 1},
|
||
],
|
||
}
|
||
payload.update(overrides)
|
||
return payload
|
||
|
||
|
||
# ── list / create / get ──────────────────────────────────────────────────
|
||
|
||
async def test_list_campaigns_empty(client, seeded_db):
|
||
resp = await client.get("/api/campaigns")
|
||
assert resp.status_code == 200
|
||
assert resp.json() == []
|
||
|
||
|
||
async def test_create_campaign_then_get(client, seeded_db):
|
||
resp = await client.post("/api/campaigns", json=_valid_payload())
|
||
assert resp.status_code == 200
|
||
body = resp.json()
|
||
campaign_id = body["id"]
|
||
assert body["status"] == "planned"
|
||
assert body["name"] == "24h-cycle"
|
||
assert body["target_id"] == "t-1"
|
||
assert body["window_seconds"] == 86400
|
||
assert body["time_scale"] == 1.0
|
||
assert len(body["plan"]) == 2
|
||
|
||
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
||
assert got["id"] == campaign_id
|
||
assert got["status"] == "planned"
|
||
assert got["plan"][0]["scenario_id"] == "s-1"
|
||
assert got["plan"][0]["offset_seconds"] == 0
|
||
assert got["plan"][0]["count"] == 2
|
||
assert got["plan"][1]["offset_seconds"] == 3600
|
||
|
||
|
||
async def test_create_campaign_default_time_scale(client, seeded_db):
|
||
payload = _valid_payload()
|
||
del payload["time_scale"]
|
||
resp = await client.post("/api/campaigns", json=payload)
|
||
assert resp.status_code == 200
|
||
assert resp.json()["time_scale"] == 1.0
|
||
|
||
|
||
async def test_list_campaigns_after_create(client, seeded_db):
|
||
await client.post("/api/campaigns", json=_valid_payload())
|
||
listing = (await client.get("/api/campaigns")).json()
|
||
assert len(listing) == 1
|
||
assert listing[0]["name"] == "24h-cycle"
|
||
|
||
|
||
async def test_list_campaigns_embeds_progress(client, seeded_db):
|
||
from agenteval.models import EvalRun, RunStatus
|
||
|
||
# _valid_payload plan totals 2 + 1 = 3 planned runs.
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
repo = RunRepository(seeded_db)
|
||
repo.create(EvalRun(
|
||
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||
status=RunStatus.COMPLETED,
|
||
summary={"pass_rate": 1.0, "avg_latency_ms": 100},
|
||
))
|
||
repo.create(EvalRun(
|
||
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||
status=RunStatus.COMPLETED,
|
||
summary={"pass_rate": 0.0, "avg_latency_ms": 200},
|
||
))
|
||
|
||
listing = (await client.get("/api/campaigns")).json()
|
||
assert len(listing) == 1
|
||
progress = listing[0]["progress"]
|
||
assert progress["planned_total"] == 3 # Σ plan.count
|
||
assert progress["completed_runs"] == 2
|
||
assert progress["overall_pass_rate"] == 0.5
|
||
|
||
|
||
async def test_get_campaign_not_found(client, seeded_db):
|
||
resp = await client.get("/api/campaigns/does-not-exist")
|
||
assert resp.status_code == 404
|
||
|
||
|
||
# ── validation ─────────────────────────────────────────────────────────────
|
||
|
||
async def test_create_campaign_empty_plan_rejected(client, seeded_db):
|
||
resp = await client.post("/api/campaigns", json=_valid_payload(plan=[]))
|
||
assert resp.status_code == 422
|
||
|
||
|
||
async def test_create_campaign_missing_target_404(client, seeded_db):
|
||
resp = await client.post("/api/campaigns", json=_valid_payload(target_id="nope"))
|
||
assert resp.status_code == 404
|
||
|
||
|
||
async def test_create_campaign_missing_scenario_404(client, seeded_db):
|
||
payload = _valid_payload(plan=[{"scenario_id": "nope", "offset_seconds": 0, "count": 1}])
|
||
resp = await client.post("/api/campaigns", json=payload)
|
||
assert resp.status_code == 404
|
||
|
||
|
||
async def test_create_campaign_bad_window_rejected(client, seeded_db):
|
||
resp = await client.post("/api/campaigns", json=_valid_payload(window_seconds=0))
|
||
assert resp.status_code == 422
|
||
|
||
|
||
async def test_create_campaign_bad_time_scale_rejected(client, seeded_db):
|
||
resp = await client.post("/api/campaigns", json=_valid_payload(time_scale=0))
|
||
assert resp.status_code == 422
|
||
|
||
|
||
# ── campaign_id column does not break existing Run behavior ─────────────────
|
||
|
||
async def test_run_campaign_id_defaults_none(seeded_db):
|
||
repo = RunRepository(seeded_db)
|
||
run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1"))
|
||
assert run.campaign_id is None
|
||
fetched = repo.get(run.id)
|
||
assert fetched.campaign_id is None
|
||
|
||
|
||
async def test_run_can_belong_to_campaign(seeded_db):
|
||
repo = RunRepository(seeded_db)
|
||
run = repo.create(EvalRun(target_id="t-1", scenario_id="s-1", campaign_id="camp-1"))
|
||
assert run.campaign_id == "camp-1"
|
||
fetched = repo.get(run.id)
|
||
assert fetched.campaign_id == "camp-1"
|
||
|
||
|
||
# ── cancel endpoint + progress in detail (ticket 03; scheduler stubbed) ─────
|
||
|
||
async def test_cancel_campaign_then_state(client, seeded_db):
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
|
||
resp = await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
||
assert resp.status_code == 200
|
||
assert resp.json()["status"] == "cancelled"
|
||
|
||
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
||
assert got["status"] == "cancelled"
|
||
|
||
|
||
async def test_cancel_already_cancelled_rejected(client, seeded_db):
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
||
again = await client.post(f"/api/campaigns/{campaign_id}/cancel")
|
||
assert again.status_code == 400
|
||
|
||
|
||
async def test_cancel_missing_campaign_404(client, seeded_db):
|
||
resp = await client.post("/api/campaigns/nope/cancel")
|
||
assert resp.status_code == 404
|
||
|
||
|
||
async def test_detail_includes_progress_fields(client, seeded_db):
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
||
assert "progress" in got
|
||
progress = got["progress"]
|
||
assert progress["spawned_runs"] == 0
|
||
assert progress["completed_runs"] == 0
|
||
assert progress["current_offset_seconds"] == 0.0
|
||
|
||
|
||
# ── campaign report endpoint (ticket 04) ─────────────────────────────────────
|
||
|
||
async def test_campaign_report_structure_and_values(client, seeded_db):
|
||
from agenteval.models import EvalRun, RunStatus
|
||
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
|
||
repo = RunRepository(seeded_db)
|
||
repo.create(EvalRun(
|
||
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||
status=RunStatus.COMPLETED,
|
||
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
||
))
|
||
repo.create(EvalRun(
|
||
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||
status=RunStatus.COMPLETED,
|
||
summary={"total_cases": 1, "passed_cases": 0, "pass_rate": 0.0, "avg_latency_ms": 200},
|
||
))
|
||
|
||
report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json()
|
||
assert report["campaign_id"] == campaign_id
|
||
assert "time_trend" in report and "capability_summary" in report
|
||
assert report["summary"]["total_runs"] == 2
|
||
assert report["summary"]["completed_runs"] == 2
|
||
assert report["summary"]["overall_pass_rate"] == 0.5
|
||
cap = next(c for c in report["capability_summary"] if c["scenario_id"] == "s-1")
|
||
assert cap["run_count"] == 2
|
||
assert cap["scenario_name"] == "mock-scenario"
|
||
|
||
|
||
async def test_campaign_report_missing_404(client, seeded_db):
|
||
resp = await client.get("/api/campaigns/nope/report")
|
||
assert resp.status_code == 404
|
||
|
||
|
||
async def test_campaign_report_markdown_export(client, seeded_db):
|
||
from agenteval.models import EvalRun, RunStatus
|
||
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
RunRepository(seeded_db).create(EvalRun(
|
||
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||
status=RunStatus.COMPLETED,
|
||
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
||
))
|
||
|
||
resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown")
|
||
assert resp.status_code == 200
|
||
assert "text/markdown" in resp.headers["content-type"]
|
||
assert "attachment" in resp.headers["content-disposition"]
|
||
assert "# 活动周期报告" in resp.text
|
||
assert "## 时间趋势" in resp.text
|
||
assert "## 能力汇总" in resp.text
|
||
|
||
|
||
# ── exploration findings in report/export (v0.9 ticket 05) ───────────────────
|
||
|
||
|
||
def _seed_exploration_session(seeded_db, campaign_id: str) -> None:
|
||
from agenteval.exploration.models import ExplorationSession
|
||
from agenteval.storage.repository import ExplorationSessionRepository
|
||
|
||
repo = ExplorationSessionRepository(seeded_db)
|
||
session_obj = repo.create(
|
||
ExplorationSession(campaign_id=campaign_id, target_id="t-1", persona={"name": "x"}, goal="缴费")
|
||
)
|
||
session_obj.experience = {
|
||
"goal_achieved": True,
|
||
"blockers": ["缴费入口难找"],
|
||
"misled": [],
|
||
"emotion": "neutral",
|
||
"notes": "",
|
||
}
|
||
repo.update(session_obj)
|
||
|
||
|
||
async def test_campaign_report_includes_exploration_findings(client, seeded_db):
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
_seed_exploration_session(seeded_db, campaign_id)
|
||
|
||
report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json()
|
||
exploration = report["exploration"]
|
||
assert exploration["session_count"] == 1
|
||
assert exploration["goal_achievement_rate"] == 1.0
|
||
assert exploration["issues"] == [{"issue": "缴费入口难找", "count": 1}]
|
||
|
||
|
||
async def test_campaign_report_without_exploration_omits_key(client, seeded_db):
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
report = (await client.get(f"/api/campaigns/{campaign_id}/report")).json()
|
||
assert "exploration" not in report
|
||
|
||
|
||
async def test_markdown_export_appends_exploration_appendix(client, seeded_db):
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
_seed_exploration_session(seeded_db, campaign_id)
|
||
|
||
resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown")
|
||
assert resp.status_code == 200
|
||
assert "## 探索发现" in resp.text
|
||
assert "缴费入口难找 ×1" in resp.text
|
||
|
||
|
||
async def test_markdown_export_without_exploration_leaves_no_trace(client, seeded_db):
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
resp = await client.get(f"/api/campaigns/{campaign_id}/report/markdown")
|
||
assert resp.status_code == 200
|
||
assert "探索发现" not in resp.text
|
||
|
||
|
||
# ── campaign timeline endpoint (ticket 06) ───────────────────────────────────
|
||
|
||
async def test_campaign_timeline_structure(client, seeded_db):
|
||
from agenteval.models import EvalRun, RunStatus
|
||
|
||
campaign_id = (await client.post("/api/campaigns", json=_valid_payload())).json()["id"]
|
||
|
||
repo = RunRepository(seeded_db)
|
||
repo.create(EvalRun(
|
||
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||
status=RunStatus.COMPLETED,
|
||
summary={"total_cases": 1, "passed_cases": 1, "pass_rate": 1.0, "avg_latency_ms": 100},
|
||
))
|
||
repo.create(EvalRun(
|
||
target_id="t-1", scenario_id="s-1", campaign_id=campaign_id,
|
||
status=RunStatus.FAILED,
|
||
))
|
||
|
||
body = (await client.get(f"/api/campaigns/{campaign_id}/timeline")).json()
|
||
assert "entries" in body
|
||
entries = body["entries"]
|
||
assert len(entries) == 2
|
||
for e in entries:
|
||
assert set(e) >= {
|
||
"run_id", "scenario_id", "scenario_name", "offset_seconds",
|
||
"status", "pass_rate", "avg_latency_ms", "started_at",
|
||
}
|
||
assert e["scenario_name"] == "mock-scenario"
|
||
passed = next(e for e in entries if e["status"] == "completed")
|
||
assert passed["pass_rate"] == 1.0
|
||
|
||
|
||
async def test_campaign_timeline_missing_404(client, seeded_db):
|
||
resp = await client.get("/api/campaigns/nope/timeline")
|
||
assert resp.status_code == 404
|
||
|
||
|
||
# ── analysis model override (v0.7 ticket 02) ─────────────────────────────
|
||
|
||
def _seed_chat_config(session, config_id: str = "mc-1") -> None:
|
||
from agenteval.storage.db import ModelConfigDB
|
||
from agenteval.storage.model_config_repository import ModelConfigRepository
|
||
|
||
ModelConfigRepository(session).create(ModelConfigDB(
|
||
id=config_id, name="chat-cfg", provider="openai_compatible", capability="chat",
|
||
endpoint_url="https://models.example.com/v1/chat/completions", model_name="chat-model",
|
||
))
|
||
|
||
|
||
async def test_create_campaign_with_analysis_model_override(client, seeded_db):
|
||
_seed_chat_config(seeded_db)
|
||
resp = await client.post("/api/campaigns", json=_valid_payload(analysis_model_config_id="mc-1"))
|
||
assert resp.status_code == 200
|
||
assert resp.json()["analysis_model_config_id"] == "mc-1"
|
||
|
||
campaign_id = resp.json()["id"]
|
||
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
||
assert got["analysis_model_config_id"] == "mc-1"
|
||
listing = (await client.get("/api/campaigns")).json()
|
||
assert listing[0]["analysis_model_config_id"] == "mc-1"
|
||
|
||
|
||
async def test_create_campaign_without_override_stores_null(client, seeded_db):
|
||
resp = await client.post("/api/campaigns", json=_valid_payload())
|
||
assert resp.status_code == 200
|
||
assert resp.json()["analysis_model_config_id"] is None
|
||
|
||
campaign_id = resp.json()["id"]
|
||
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
||
assert got["analysis_model_config_id"] is None
|
||
|
||
|
||
async def test_create_campaign_invalid_analysis_model_400(client, seeded_db):
|
||
resp = await client.post("/api/campaigns", json=_valid_payload(analysis_model_config_id="nope"))
|
||
assert resp.status_code == 400
|
||
|
||
|
||
async def test_create_campaign_with_exploration_config(client, seeded_db):
|
||
payload = _valid_payload(
|
||
exploration_seeds={"personas": ["急性子用户", "谨慎的老年用户"], "goals": ["查询账单并缴费", "修改收货地址"]},
|
||
exploration_budget={"max_sessions": 4, "max_turns": 6, "min_interval_seconds": 3600},
|
||
)
|
||
resp = await client.post("/api/campaigns", json=payload)
|
||
assert resp.status_code == 200, resp.text
|
||
body = resp.json()
|
||
assert body["exploration_seeds"] == {
|
||
"personas": ["急性子用户", "谨慎的老年用户"],
|
||
"goals": ["查询账单并缴费", "修改收货地址"],
|
||
}
|
||
assert body["exploration_budget"] == {"max_sessions": 4, "max_turns": 6, "min_interval_seconds": 3600}
|
||
|
||
got = (await client.get(f"/api/campaigns/{body['id']}")).json()
|
||
assert got["exploration_seeds"]["personas"] == ["急性子用户", "谨慎的老年用户"]
|
||
assert got["exploration_budget"]["max_turns"] == 6
|
||
|
||
|
||
async def test_create_campaign_without_exploration_config(client, seeded_db):
|
||
body = (await client.post("/api/campaigns", json=_valid_payload())).json()
|
||
assert body["exploration_seeds"] is None
|
||
assert body["exploration_budget"] is None
|
||
|
||
|
||
async def test_empty_seeds_campaign_opts_out_of_exploration(client, seeded_db):
|
||
payload = _valid_payload(exploration_seeds={"personas": [], "goals": []})
|
||
body = (await client.post("/api/campaigns", json=payload)).json()
|
||
assert body["exploration_seeds"] is None
|
||
|
||
|
||
def test_exploration_config_migration_on_existing_db(tmp_path, monkeypatch):
|
||
"""The two campaign config columns apply on a DB at the previous head."""
|
||
from pathlib import Path
|
||
|
||
from agenteval.storage import db as db_module
|
||
from alembic import command
|
||
from alembic.config import Config
|
||
from sqlalchemy import create_engine, inspect, text
|
||
from sqlmodel import SQLModel
|
||
|
||
database_url = f"sqlite:///{tmp_path / 'campaign_config.db'}"
|
||
monkeypatch.setattr(db_module, "DATABASE_URL", database_url)
|
||
config = Config(str(Path(__file__).resolve().parents[2] / "alembic.ini"))
|
||
|
||
SQLModel.metadata.create_all(create_engine(database_url))
|
||
with create_engine(database_url).begin() as connection:
|
||
connection.execute(text("DROP TABLE IF EXISTS intelligent_eval_messages"))
|
||
connection.execute(text("DROP TABLE IF EXISTS intelligent_eval_sessions"))
|
||
connection.execute(text("DROP TABLE IF EXISTS intelligent_evals"))
|
||
connection.execute(text("DROP TABLE IF EXISTS exploration_sessions"))
|
||
connection.execute(text("DROP TABLE IF EXISTS exploration_messages"))
|
||
connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_seeds"))
|
||
connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_budget"))
|
||
connection.execute(text("ALTER TABLE campaigns DROP COLUMN last_patrolled_at"))
|
||
connection.execute(text("DROP TABLE IF EXISTS alembic_version"))
|
||
|
||
command.stamp(config, "0e4a7c91d2b3")
|
||
command.upgrade(config, "head")
|
||
|
||
cols = {c["name"] for c in inspect(create_engine(database_url)).get_columns("campaigns")}
|
||
assert {"exploration_seeds", "exploration_budget"} <= cols
|