3705945a7d
架构重构（候选 1-6）： - storage/repository.py 按域拆分为包（target/scenario/run/campaign/result） - storage/db.py 按域拆分为包（eval/campaign/file/model_config/intelligent_eval） - intelligent_eval/lifecycle.py 按状态机阶段拆分为包 - services/runs.py 编排逻辑下沉 - Campaigns.tsx 拆分为 campaigns/ 子组件 测试补全（候选 7）： 前端（+125 用例，107→232）： - utils/ 纯函数：date/campaignTime/ruleLabels/fileTree/fileFormat/colors - stores/tabStore 状态管理 - 核心组件：FormDrawer/PageWrapper/ChatBubble/GeneratedMessages/SectionHeader/StatCard/TurnList - 业务组件：CaseBlock/CaseDetail/RuleOverview/WindowTimeline/RunList/TabBar/CampaignRunTimeline - 文件管理：FileCategoryTree/FileTable - hooks：sessionReducer/useFiles/useRunSession 后端（+38 用例，916→954）： - targets API CRUD + 404 路径 - WebSocket 连接管理器 - proxy 头部重写（CSP/X-Frame-Options） - target 仓储 update 方法 - app 健康检查 + SPA 404 - scenarios 模板端点 + 404 - files API 边缘分支（404 场景 + 500 兜底） - files service update_category - 智能评估状态机迁移测试 门禁状态： - 前端：tsc 干净 + 232 passed - 后端：954 passed + ruff 全绿
333 lines
12 KiB
Python
333 lines
12 KiB
Python
"""Integration tests for the /api/runs endpoints.
|
|
|
|
Uses ``httpx.AsyncClient`` with ``app=`` to drive the FastAPI app in-process
|
|
(no real server). The engine's channel is monkeypatched to a MockChannel so
|
|
no network calls are made.
|
|
"""
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
from agenteval.models import (
|
|
Case,
|
|
CaseType,
|
|
ChannelType,
|
|
EvalTarget,
|
|
PlatformType,
|
|
Scenario,
|
|
TargetStatus,
|
|
)
|
|
from agenteval.storage.repository import ScenarioRepository, TargetRepository
|
|
from agenteval.web.app import app
|
|
from httpx import ASGITransport, AsyncClient
|
|
|
|
from tests.unit.mock_channel import MockChannel
|
|
|
|
|
|
@pytest.fixture()
|
|
def seeded_db(db_session, monkeypatch):
|
|
"""Patch the global get_session to return the test session, then seed a
|
|
target + scenario. Also skip init_db so the real data/ DB is not touched."""
|
|
from agenteval.storage import db as db_module
|
|
from agenteval.web import app as app_module
|
|
|
|
# Skip init_db in lifespan (it would create tables in the real data/ DB).
|
|
monkeypatch.setattr(app_module, "init_db", lambda: None)
|
|
|
|
# Background task (services.runs.execute_run) calls get_session() directly.
|
|
# Because `from agenteval.storage.db import get_session` binds a local
|
|
# reference in every importing module, we must patch every consumer.
|
|
def _test_get_session():
|
|
return db_session
|
|
|
|
from agenteval.evaluation import engine as engine_module
|
|
from agenteval.services import runs as run_service_module
|
|
from agenteval.storage import repository as repo_module
|
|
|
|
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(run_service_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(engine_module, "get_session", _test_get_session)
|
|
|
|
# FastAPI endpoints use Depends(get_db). Override the dependency.
|
|
from agenteval.web.deps import get_db
|
|
|
|
def _test_get_db():
|
|
try:
|
|
yield db_session
|
|
finally:
|
|
pass # Don't close the fixture-owned session.
|
|
|
|
app.dependency_overrides[get_db] = _test_get_db
|
|
|
|
target = EvalTarget(
|
|
id="t-1", name="mock-target",
|
|
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={
|
|
"base_url": "http://mock", "token": "x",
|
|
"tenant": "t", "chat_channel_id": "c", "chat_contact_id": "u",
|
|
},
|
|
status=TargetStatus.ACTIVE,
|
|
)
|
|
TargetRepository(db_session).create(target)
|
|
|
|
scenario = Scenario(
|
|
id="s-1", name="mock-scenario",
|
|
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
|
)
|
|
ScenarioRepository(db_session).create(scenario)
|
|
|
|
yield db_session
|
|
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
@pytest.fixture()
|
|
def mock_channel(monkeypatch):
|
|
"""Make ChannelFactory.create return a MockChannel for every target."""
|
|
channel = MockChannel(reply_delay=0.02)
|
|
|
|
def _fake_create(target):
|
|
return channel
|
|
|
|
from agenteval.channels import factory as factory_module
|
|
monkeypatch.setattr(factory_module.ChannelFactory, "create", _fake_create)
|
|
return channel
|
|
|
|
|
|
@pytest.fixture()
|
|
async def client():
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
# ── list / get ───────────────────────────────────────────────────────────
|
|
|
|
async def test_list_runs_empty(client, seeded_db):
|
|
resp = await client.get("/api/runs")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
|
|
async def test_start_run_then_get(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1",
|
|
})
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
run_id = body["id"]
|
|
assert body["status"] == "pending"
|
|
|
|
# Wait for the background task to finish.
|
|
for _ in range(50):
|
|
await asyncio.sleep(0.05)
|
|
r = await client.get(f"/api/runs/{run_id}")
|
|
if r.json()["status"] in ("completed", "failed"):
|
|
break
|
|
|
|
final = (await client.get(f"/api/runs/{run_id}")).json()
|
|
assert final["status"] == "completed"
|
|
assert final["summary"]["total_cases"] == 1
|
|
assert mock_channel.send_calls == 1
|
|
|
|
|
|
async def test_cancel_run(client, seeded_db, mock_channel):
|
|
# Seed a multi-case scenario so the run takes long enough to cancel.
|
|
from agenteval.models import Case, CaseType, Scenario
|
|
from agenteval.storage.repository import ScenarioRepository
|
|
multi = Scenario(
|
|
id="s-long", name="long-scenario",
|
|
cases=[
|
|
Case(id=f"lc{i}", type=CaseType.SINGLE, messages=[f"m{i}"])
|
|
for i in range(5)
|
|
],
|
|
)
|
|
ScenarioRepository(seeded_db).create(multi)
|
|
|
|
# Use a slow channel so we have time to cancel mid-flight.
|
|
mock_channel.reply_delay = 0.1
|
|
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-long",
|
|
})
|
|
run_id = resp.json()["id"]
|
|
|
|
# Give the task a moment to start and process at least one case.
|
|
await asyncio.sleep(0.15)
|
|
|
|
cancel = await client.post(f"/api/runs/{run_id}/cancel")
|
|
assert cancel.status_code == 200
|
|
|
|
# Wait for the task to observe the cancel.
|
|
for _ in range(50):
|
|
await asyncio.sleep(0.05)
|
|
r = await client.get(f"/api/runs/{run_id}")
|
|
if r.json()["status"] == "failed":
|
|
break
|
|
|
|
final = (await client.get(f"/api/runs/{run_id}")).json()
|
|
assert final["status"] == "failed"
|
|
assert final["summary"]["error"]["code"] == "cancelled_by_user"
|
|
# Not all 5 cases should have run.
|
|
assert mock_channel.send_calls < 5
|
|
|
|
|
|
async def test_get_run_logs(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1",
|
|
})
|
|
run_id = resp.json()["id"]
|
|
|
|
for _ in range(50):
|
|
await asyncio.sleep(0.05)
|
|
r = await client.get(f"/api/runs/{run_id}")
|
|
if r.json()["status"] == "completed":
|
|
break
|
|
|
|
logs = (await client.get(f"/api/runs/{run_id}/logs")).json()
|
|
assert set(logs.keys()) == {"turns", "results", "case_verdicts", "scenario_snapshot"}
|
|
|
|
# Characterization: exact turn shape, round-tripping the sent message.
|
|
assert len(logs["turns"]) == 1
|
|
turn = logs["turns"][0]
|
|
assert set(turn.keys()) == {
|
|
"id", "case_id", "round_index", "latency_ms",
|
|
"sent_text", "reply_text", "sent_at", "received_at",
|
|
}
|
|
assert turn["case_id"] == "c1"
|
|
assert turn["round_index"] == 1
|
|
assert turn["sent_text"] == "hi"
|
|
assert turn["reply_text"] # MockChannel always replies
|
|
assert turn["latency_ms"] is not None
|
|
assert turn["sent_at"] and turn["received_at"]
|
|
|
|
# case_verdicts are keyed by case id with passed/connectivity flags.
|
|
assert set(logs["case_verdicts"].keys()) == {"c1"}
|
|
assert set(logs["case_verdicts"]["c1"].keys()) == {"passed", "connectivity"}
|
|
|
|
# scenario_snapshot mirrors the seeded case's structure.
|
|
snap = logs["scenario_snapshot"]["c1"]
|
|
assert snap["id"] == "c1"
|
|
assert snap["type"] == "single"
|
|
assert snap["messages"] == ["hi"]
|
|
assert set(snap["expectations"].keys()) == {
|
|
"intent", "keywords_include", "keywords_exclude",
|
|
"response_time_max_ms", "coherence_min_score",
|
|
}
|
|
assert isinstance(snap["eval_rules"], list)
|
|
|
|
# results entries, when present, carry the rule outcome shape.
|
|
for entry in logs["results"]:
|
|
assert set(entry.keys()) == {"case_id", "rule_type", "passed", "score", "reason"}
|
|
|
|
|
|
async def test_start_run_missing_target(client, seeded_db):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "does-not-exist", "scenario_id": "s-1",
|
|
})
|
|
assert resp.status_code == 404
|
|
|
|
|
|
async def test_cancel_run_without_live_task_marks_db_failed(client, seeded_db):
|
|
"""No live task (e.g. process restarted): cancel falls back to the DB row."""
|
|
from agenteval.models import EvalRun
|
|
from agenteval.storage.repository import RunRepository
|
|
|
|
run = RunRepository(seeded_db).create(EvalRun(target_id="t-1", scenario_id="s-1"))
|
|
|
|
resp = await client.post(f"/api/runs/{run.id}/cancel")
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["status"] == "failed"
|
|
assert body["summary"]["error"]["code"] == "cancelled_by_user"
|
|
|
|
# Persisted, not just echoed.
|
|
got = (await client.get(f"/api/runs/{run.id}")).json()
|
|
assert got["status"] == "failed"
|
|
|
|
|
|
async def test_cancel_finished_run_rejected(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1",
|
|
})
|
|
run_id = resp.json()["id"]
|
|
for _ in range(50):
|
|
await asyncio.sleep(0.05)
|
|
r = await client.get(f"/api/runs/{run_id}")
|
|
if r.json()["status"] == "completed":
|
|
break
|
|
|
|
resp = await client.post(f"/api/runs/{run_id}/cancel")
|
|
assert resp.status_code == 400
|
|
|
|
|
|
# ── triggered_by ─────────────────────────────────────────────────────────
|
|
|
|
async def test_start_run_default_triggered_by_manual(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1",
|
|
})
|
|
assert resp.status_code == 200
|
|
assert resp.json()["triggered_by"] == "manual"
|
|
|
|
|
|
async def test_start_run_ai_assistant_triggered_by(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1", "triggered_by": "ai_assistant",
|
|
})
|
|
assert resp.status_code == 200
|
|
run_id = resp.json()["id"]
|
|
assert resp.json()["triggered_by"] == "ai_assistant"
|
|
|
|
# Persisted, not just echoed.
|
|
got = (await client.get(f"/api/runs/{run_id}")).json()
|
|
assert got["triggered_by"] == "ai_assistant"
|
|
|
|
|
|
async def test_start_run_invalid_triggered_by_422(client, seeded_db):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1", "triggered_by": "robot",
|
|
})
|
|
assert resp.status_code == 422
|
|
|
|
|
|
async def test_list_runs_includes_names_and_trigger(client, seeded_db, mock_channel):
|
|
await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1", "triggered_by": "ai_assistant",
|
|
})
|
|
listing = (await client.get("/api/runs")).json()
|
|
assert len(listing) == 1
|
|
row = listing[0]
|
|
assert row["scenario_name"] == "mock-scenario"
|
|
assert row["target_name"] == "mock-target"
|
|
assert row["triggered_by"] == "ai_assistant"
|
|
|
|
|
|
# ── scenario_version snapshot (ticket 04) ────────────────────────────────
|
|
|
|
async def test_start_run_snapshots_scenario_version(client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1",
|
|
})
|
|
assert resp.status_code == 200
|
|
assert resp.json()["scenario_version"] == 1
|
|
|
|
# 编辑考纲 → 场景升版 → 新运行快照新版本；旧运行保持 1
|
|
old_run_id = resp.json()["id"]
|
|
scenario = ScenarioRepository(seeded_db).get("s-1")
|
|
scenario.cases.append(Case(id="c2", type=CaseType.SINGLE, messages=["more"]))
|
|
ScenarioRepository(seeded_db).update(scenario)
|
|
|
|
resp2 = await client.post("/api/runs", json={
|
|
"target_id": "t-1", "scenario_id": "s-1", "triggered_by": "ai_assistant",
|
|
})
|
|
assert resp2.json()["scenario_version"] == 2
|
|
|
|
old = (await client.get(f"/api/runs/{old_run_id}")).json()
|
|
assert old["scenario_version"] == 1
|
|
|
|
listing = (await client.get("/api/runs")).json()
|
|
assert {row["scenario_version"] for row in listing} == {1, 2}
|