0a47260237
运行创建时快照场景考纲版本，三种触发来源（手动/AI 助手/CLI）一致；
迁移回填存量运行为其场景当前版本，孤儿运行回填 1。运行列表、
报告头与对比卡片展示 v{n} 版本标签。
268 lines
9.3 KiB
Python
268 lines
9.3 KiB
Python
"""Integration tests for the /api/runs endpoints.
|
|
|
|
Uses ``httpx.AsyncClient`` with ``app=`` to drive the FastAPI app in-process
|
|
(no real server). The engine's channel is monkeypatched to a MockChannel so
|
|
no network calls are made.
|
|
"""
|
|
|
|
import asyncio
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
|
|
from agenteval.channels.base import SendResult
|
|
from agenteval.models import (
|
|
Case, CaseType, ChannelType, EvalTarget, PlatformType,
|
|
Scenario, TargetStatus,
|
|
)
|
|
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
|
from agenteval.web.app import app
|
|
from tests.unit.mock_channel import MockChannel
|
|
|
|
|
|
@pytest.fixture()
|
|
def seeded_db(db_session, monkeypatch):
|
|
"""Patch the global get_session to return the test session, then seed a
|
|
target + scenario. Also skip init_db so the real data/ DB is not touched."""
|
|
from agenteval.storage import db as db_module
|
|
from agenteval.web import app as app_module
|
|
|
|
# Skip init_db in lifespan (it would create tables in the real data/ DB).
|
|
monkeypatch.setattr(app_module, "init_db", lambda: None)
|
|
|
|
# Background task (_run_evaluation) calls get_session() directly.
|
|
# Because `from agenteval.storage.db import get_session` binds a local
|
|
# reference in every importing module, we must patch every consumer.
|
|
def _test_get_session():
|
|
return db_session
|
|
|
|
from agenteval.storage import db as db_module
|
|
from agenteval.web.routers import runs as runs_module
|
|
from agenteval.storage import repository as repo_module
|
|
from agenteval.evaluation import engine as engine_module
|
|
|
|
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(runs_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(engine_module, "get_session", _test_get_session)
|
|
|
|
# FastAPI endpoints use Depends(get_db). Override the dependency.
|
|
from agenteval.web.deps import get_db
|
|
|
|
def _test_get_db():
|
|
try:
|
|
yield db_session
|
|
finally:
|
|
pass # Don't close the fixture-owned session.
|
|
|
|
app.dependency_overrides[get_db] = _test_get_db
|
|
|
|
target = EvalTarget(
|
|
id="t-1", name="mock-target",
|
|
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={
|
|
"base_url": "http://mock", "token": "x",
|
|
"tenant": "t", "chat_channel_id": "c", "chat_contact_id": "u",
|
|
},
|
|
status=TargetStatus.ACTIVE,
|
|
)
|
|
TargetRepository(db_session).create(target)
|
|
|
|
scenario = Scenario(
|
|
id="s-1", name="mock-scenario",
|
|
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
|
)
|
|
ScenarioRepository(db_session).create(scenario)
|
|
|
|
yield db_session
|
|
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
@pytest.fixture()
|
|
def mock_channel(monkeypatch):
|
|
"""Make ChannelFactory.create return a MockChannel for every target."""
|
|
channel = MockChannel(reply_delay=0.02)
|
|
|
|
def _fake_create(target):
|
|
return channel
|
|
|
|
from agenteval.channels import factory as factory_module
|
|
monkeypatch.setattr(factory_module.ChannelFactory, "create", _fake_create)
|
|
return channel
|
|
|
|
|
|
@pytest.fixture()
|
|
async def client():
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
# ── list / get ───────────────────────────────────────────────────────────
|
|
|
|
async def test_list_runs_empty(client, seeded_db):
|
|
resp = await client.get("/api/runs")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
async def test_start_run_then_get(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1",
|
|
})
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
run_id = body["id"]
|
|
assert body["status"] == "pending"
|
|
|
|
# Wait for the background task to finish.
|
|
for _ in range(50):
|
|
await asyncio.sleep(0.05)
|
|
r = await client.get(f"/api/runs/{run_id}")
|
|
if r.json()["status"] in ("completed", "failed"):
|
|
break
|
|
|
|
final = (await client.get(f"/api/runs/{run_id}")).json()
|
|
assert final["status"] == "completed"
|
|
assert final["summary"]["total_cases"] == 1
|
|
assert mock_channel.send_calls == 1
|
|
|
|
|
|
async def test_cancel_run(client, seeded_db, mock_channel):
|
|
# Seed a multi-case scenario so the run takes long enough to cancel.
|
|
from agenteval.models import Case, CaseType, Scenario
|
|
from agenteval.storage.repository import ScenarioRepository
|
|
multi = Scenario(
|
|
id="s-long", name="long-scenario",
|
|
cases=[
|
|
Case(id=f"lc{i}", type=CaseType.SINGLE, messages=[f"m{i}"])
|
|
for i in range(5)
|
|
],
|
|
)
|
|
ScenarioRepository(seeded_db).create(multi)
|
|
|
|
# Use a slow channel so we have time to cancel mid-flight.
|
|
mock_channel.reply_delay = 0.1
|
|
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-long",
|
|
})
|
|
run_id = resp.json()["id"]
|
|
|
|
# Give the task a moment to start and process at least one case.
|
|
await asyncio.sleep(0.15)
|
|
|
|
cancel = await client.post(f"/api/runs/{run_id}/cancel")
|
|
assert cancel.status_code == 200
|
|
|
|
# Wait for the task to observe the cancel.
|
|
for _ in range(50):
|
|
await asyncio.sleep(0.05)
|
|
r = await client.get(f"/api/runs/{run_id}")
|
|
if r.json()["status"] == "failed":
|
|
break
|
|
|
|
final = (await client.get(f"/api/runs/{run_id}")).json()
|
|
assert final["status"] == "failed"
|
|
assert final["summary"]["error"]["code"] == "cancelled_by_user"
|
|
# Not all 5 cases should have run.
|
|
assert mock_channel.send_calls < 5
|
|
|
|
|
|
async def test_get_run_logs(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1",
|
|
})
|
|
run_id = resp.json()["id"]
|
|
|
|
for _ in range(50):
|
|
await asyncio.sleep(0.05)
|
|
r = await client.get(f"/api/runs/{run_id}")
|
|
if r.json()["status"] == "completed":
|
|
break
|
|
|
|
logs = (await client.get(f"/api/runs/{run_id}/logs")).json()
|
|
assert "turns" in logs
|
|
assert "results" in logs
|
|
assert "scenario_snapshot" in logs
|
|
assert len(logs["turns"]) == 1
|
|
|
|
|
|
async def test_start_run_missing_target(client, seeded_db):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "does-not-exist", "scenario_id": "s-1",
|
|
})
|
|
assert resp.status_code == 404
|
|
|
|
|
|
# ── triggered_by ─────────────────────────────────────────────────────────
|
|
|
|
async def test_start_run_default_triggered_by_manual(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1",
|
|
})
|
|
assert resp.status_code == 200
|
|
assert resp.json()["triggered_by"] == "manual"
|
|
|
|
|
|
async def test_start_run_ai_assistant_triggered_by(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1", "triggered_by": "ai_assistant",
|
|
})
|
|
assert resp.status_code == 200
|
|
run_id = resp.json()["id"]
|
|
assert resp.json()["triggered_by"] == "ai_assistant"
|
|
|
|
# Persisted, not just echoed.
|
|
got = (await client.get(f"/api/runs/{run_id}")).json()
|
|
assert got["triggered_by"] == "ai_assistant"
|
|
|
|
|
|
async def test_start_run_invalid_triggered_by_422(client, seeded_db):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1", "triggered_by": "robot",
|
|
})
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_list_runs_includes_names_and_trigger(client, seeded_db, mock_channel):
|
|
await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1", "triggered_by": "ai_assistant",
|
|
})
|
|
listing = (await client.get("/api/runs")).json()
|
|
assert len(listing) == 1
|
|
row = listing[0]
|
|
assert row["scenario_name"] == "mock-scenario"
|
|
assert row["target_name"] == "mock-target"
|
|
assert row["triggered_by"] == "ai_assistant"
|
|
|
|
|
|
# ── scenario_version snapshot (ticket 04) ────────────────────────────────
|
|
|
|
async def test_start_run_snapshots_scenario_version(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1",
|
|
})
|
|
assert resp.status_code == 200
|
|
assert resp.json()["scenario_version"] == 1
|
|
|
|
# 编辑考纲 → 场景升版 → 新运行快照新版本；旧运行保持 1
|
|
old_run_id = resp.json()["id"]
|
|
scenario = ScenarioRepository(seeded_db).get("s-1")
|
|
scenario.cases.append(Case(id="c2", type=CaseType.SINGLE, messages=["more"]))
|
|
ScenarioRepository(seeded_db).update(scenario)
|
|
|
|
resp2 = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1", "triggered_by": "ai_assistant",
|
|
})
|
|
assert resp2.json()["scenario_version"] == 2
|
|
|
|
old = (await client.get(f"/api/runs/{old_run_id}")).json()
|
|
assert old["scenario_version"] == 1
|
|
|
|
listing = (await client.get("/api/runs")).json()
|
|
assert {row["scenario_version"] for row in listing} == {1, 2}
|