544 lines
22 KiB
Python
544 lines
22 KiB
Python
"""Integration tests for intelligent eval lifecycle + session + report API (tickets 02, 03, 04).
|
|
|
|
Uses httpx.AsyncClient with app= to drive the FastAPI app in-process.
|
|
Covers the full lifecycle (create → plan → approve → execute → cancel),
|
|
reject/resubmit flow, illegal transition rejections (409), the session
|
|
lifecycle (create → message → close) with channel I/O stubbed via MockChannel,
|
|
and the report flow (submit → completed → read → Markdown export).
|
|
"""
|
|
|
|
import pytest
|
|
from agenteval.models import ChannelType, EvalTarget, PlatformType, TargetStatus
|
|
from agenteval.storage.repository import TargetRepository
|
|
from agenteval.web.app import app
|
|
from httpx import ASGITransport, AsyncClient
|
|
|
|
|
|
@pytest.fixture()
|
|
def seeded_db(db_session, monkeypatch):
|
|
"""Patch get_session/get_db to the test session and seed a target."""
|
|
from agenteval.storage import db as db_module
|
|
from agenteval.storage import repository as repo_module
|
|
from agenteval.web import app as app_module
|
|
|
|
monkeypatch.setattr(app_module, "init_db", lambda: None)
|
|
|
|
def _test_get_session():
|
|
return db_session
|
|
|
|
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
|
|
|
from agenteval.web.deps import get_db
|
|
|
|
def _test_get_db():
|
|
try:
|
|
yield db_session
|
|
finally:
|
|
pass
|
|
|
|
app.dependency_overrides[get_db] = _test_get_db
|
|
|
|
target = EvalTarget(
|
|
id="t-1",
|
|
name="mock-target",
|
|
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={"base_url": "http://mock", "token": "x"},
|
|
status=TargetStatus.ACTIVE,
|
|
)
|
|
TargetRepository(db_session).create(target)
|
|
|
|
yield db_session
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
@pytest.fixture()
|
|
async def client():
|
|
transport = ASGITransport(app=app)
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
async def _create_eval(client, **overrides) -> dict:
|
|
payload = {
|
|
"name": "客服助手智能评估",
|
|
"target_id": "t-1",
|
|
"goal": "评估退货流程处理能力",
|
|
"seeds": {"personas": ["急躁老客户"], "goals": ["完成退货"]},
|
|
"intent": "考察退货全流程",
|
|
"role_description": "模拟真实用户",
|
|
"time_window_hours": 24,
|
|
}
|
|
payload.update(overrides)
|
|
resp = await client.post("/api/intelligent-evals", json=payload)
|
|
assert resp.status_code == 200, resp.text
|
|
return resp.json()
|
|
|
|
|
|
async def _submit_plan(client, eval_id: str, plan: dict | None = None) -> dict:
|
|
if plan is None:
|
|
plan = {
|
|
"dimensions": ["退货流程", "投诉处理"],
|
|
"virtual_users": [{"persona": {"background": "老客户"}, "goal": "完成退货"}],
|
|
"time_distribution": [{"time_slot": "0-2h", "sessions": 1, "scenario": "早间咨询"}],
|
|
"estimated_sessions": 3,
|
|
"budget": {"max_turns_per_session": 12, "total_max_turns": 36},
|
|
"completion_criteria": "每个维度至少一个会话",
|
|
}
|
|
resp = await client.put(f"/api/intelligent-evals/{eval_id}/plan", json={"plan": plan})
|
|
assert resp.status_code == 200, resp.text
|
|
return resp.json()
|
|
|
|
|
|
class TestCreateEval:
|
|
async def test_create_transitions_to_planning(self, client, seeded_db):
|
|
data = await _create_eval(client)
|
|
assert data["status"] == "planning"
|
|
assert data["name"] == "客服助手智能评估"
|
|
assert data["goal"] == "评估退货流程处理能力"
|
|
assert data["time_window_hours"] == 24
|
|
assert "id" in data
|
|
|
|
async def test_create_requires_name(self, client, seeded_db):
|
|
resp = await client.post(
|
|
"/api/intelligent-evals",
|
|
json={
|
|
"name": "",
|
|
"target_id": "t-1",
|
|
"goal": "test",
|
|
},
|
|
)
|
|
assert resp.status_code == 422
|
|
|
|
async def test_create_requires_goal(self, client, seeded_db):
|
|
resp = await client.post(
|
|
"/api/intelligent-evals",
|
|
json={
|
|
"name": "test",
|
|
"target_id": "t-1",
|
|
"goal": "",
|
|
},
|
|
)
|
|
assert resp.status_code == 422
|
|
|
|
async def test_create_requires_existing_target(self, client, seeded_db):
|
|
resp = await client.post(
|
|
"/api/intelligent-evals",
|
|
json={"name": "test", "target_id": "missing", "goal": "evaluate"},
|
|
)
|
|
assert resp.status_code == 404
|
|
|
|
|
|
class TestPlanApproval:
|
|
async def test_submit_plan_transitions_to_pending_approval(self, client, seeded_db):
|
|
ev = await _create_eval(client)
|
|
data = await _submit_plan(client, ev["id"])
|
|
assert data["status"] == "pending_approval"
|
|
assert data["plan"]["dimensions"] == ["退货流程", "投诉处理"]
|
|
|
|
async def test_approve_transitions_to_executing(self, client, seeded_db):
|
|
ev = await _create_eval(client)
|
|
await _submit_plan(client, ev["id"])
|
|
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["status"] == "executing"
|
|
assert data["started_at"] is not None
|
|
|
|
async def test_reject_transitions_back_to_planning(self, client, seeded_db):
|
|
ev = await _create_eval(client)
|
|
await _submit_plan(client, ev["id"])
|
|
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/reject", json={"feedback": "缺少投诉维度"})
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["status"] == "planning"
|
|
assert data["plan_feedback"] == "缺少投诉维度"
|
|
|
|
async def test_reject_then_resubmit(self, client, seeded_db):
|
|
ev = await _create_eval(client)
|
|
await _submit_plan(client, ev["id"], plan={"dimensions": ["A"]})
|
|
await client.post(f"/api/intelligent-evals/{ev['id']}/reject", json={"feedback": "加B"})
|
|
data = await _submit_plan(client, ev["id"], plan={"dimensions": ["A", "B"]})
|
|
assert data["status"] == "pending_approval"
|
|
assert data["plan"]["dimensions"] == ["A", "B"]
|
|
assert data["plan_feedback"] is None
|
|
|
|
|
|
class TestCancel:
|
|
async def _create_executing(self, client) -> str:
|
|
ev = await _create_eval(client)
|
|
await _submit_plan(client, ev["id"])
|
|
await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
|
return ev["id"]
|
|
|
|
async def test_cancel_executing(self, client, seeded_db):
|
|
eval_id = await self._create_executing(client)
|
|
resp = await client.post(f"/api/intelligent-evals/{eval_id}/cancel")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["status"] == "cancelled"
|
|
assert data["completed_at"] is not None
|
|
|
|
async def test_cancel_pending_approval(self, client, seeded_db):
|
|
ev = await _create_eval(client)
|
|
await _submit_plan(client, ev["id"])
|
|
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/cancel")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["status"] == "cancelled"
|
|
|
|
|
|
class TestIllegalTransitions:
|
|
async def test_approve_from_planning_returns_409(self, client, seeded_db):
|
|
ev = await _create_eval(client)
|
|
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
|
assert resp.status_code == 409
|
|
|
|
async def test_double_cancel_returns_409(self, client, seeded_db):
|
|
ev = await _create_eval(client)
|
|
await _submit_plan(client, ev["id"])
|
|
await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
|
await client.post(f"/api/intelligent-evals/{ev['id']}/cancel")
|
|
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/cancel")
|
|
assert resp.status_code == 409
|
|
|
|
async def test_submit_plan_from_executing_returns_409(self, client, seeded_db):
|
|
ev = await _create_eval(client)
|
|
await _submit_plan(client, ev["id"])
|
|
await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
|
resp = await client.put(f"/api/intelligent-evals/{ev['id']}/plan", json={"plan": {"new": True}})
|
|
assert resp.status_code == 409
|
|
|
|
async def test_not_found_returns_404(self, client, seeded_db):
|
|
resp = await client.get("/api/intelligent-evals/nonexistent")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
class TestListAndGet:
|
|
async def test_list_empty(self, client, seeded_db):
|
|
resp = await client.get("/api/intelligent-evals")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["intelligent_evals"] == []
|
|
|
|
async def test_list_returns_created(self, client, seeded_db):
|
|
await _create_eval(client, name="e1")
|
|
await _create_eval(client, name="e2")
|
|
resp = await client.get("/api/intelligent-evals")
|
|
assert len(resp.json()["intelligent_evals"]) == 2
|
|
|
|
async def test_get_by_id(self, client, seeded_db):
|
|
ev = await _create_eval(client, name="e1")
|
|
resp = await client.get(f"/api/intelligent-evals/{ev['id']}")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["name"] == "e1"
|
|
|
|
|
|
async def _create_executing_eval(client) -> str:
|
|
ev = await _create_eval(client)
|
|
await _submit_plan(client, ev["id"])
|
|
await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
|
return ev["id"]
|
|
|
|
|
|
async def _create_session(client, eval_id: str, **overrides) -> dict:
|
|
payload = {
|
|
"persona": {"background": "急躁老客户", "style": "直接"},
|
|
"goal": "完成退货",
|
|
"dimension": "退货流程",
|
|
}
|
|
payload.update(overrides)
|
|
resp = await client.post(f"/api/intelligent-evals/{eval_id}/sessions", json=payload)
|
|
assert resp.status_code == 200, resp.text
|
|
return resp.json()
|
|
|
|
|
|
class TestCreateSession:
|
|
async def test_create_session_in_executing(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
data = await _create_session(client, eval_id)
|
|
assert data["eval_id"] == eval_id
|
|
assert data["target_id"] == "t-1"
|
|
assert data["status"] == "running"
|
|
assert data["persona"]["background"] == "急躁老客户"
|
|
assert data["dimension"] == "退货流程"
|
|
assert data["turn_count"] == 0
|
|
|
|
async def test_create_session_in_planning_returns_409(self, client, seeded_db):
|
|
ev = await _create_eval(client)
|
|
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/sessions", json={"goal": "g"})
|
|
assert resp.status_code == 409
|
|
|
|
async def test_create_session_requires_goal(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
resp = await client.post(f"/api/intelligent-evals/{eval_id}/sessions", json={"goal": ""})
|
|
assert resp.status_code == 422
|
|
|
|
async def test_create_session_unknown_eval_returns_404(self, client, seeded_db):
|
|
resp = await client.post("/api/intelligent-evals/nope/sessions", json={"goal": "g"})
|
|
assert resp.status_code == 404
|
|
|
|
async def test_eval_response_counts_sessions(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
await _create_session(client, eval_id)
|
|
resp = await client.get(f"/api/intelligent-evals/{eval_id}")
|
|
data = resp.json()
|
|
assert data["session_count"] == 1
|
|
assert data["completed_sessions"] == 0
|
|
assert len(data["sessions"]) == 1
|
|
assert "messages" not in data["sessions"][0]
|
|
|
|
|
|
class TestConductTurn:
|
|
@pytest.fixture()
|
|
def mock_channel(self, monkeypatch):
|
|
from agenteval.intelligent_eval import lifecycle as lifecycle_module
|
|
|
|
from tests.unit.mock_channel import MockChannel
|
|
|
|
channel = MockChannel(reply_text="好的，已为您发起退货申请")
|
|
|
|
class _StubFactory:
|
|
@staticmethod
|
|
def create(target):
|
|
return channel
|
|
|
|
monkeypatch.setattr(lifecycle_module, "ChannelFactory", _StubFactory)
|
|
return channel
|
|
|
|
async def test_turn_round_trip_persists_messages(self, client, seeded_db, mock_channel):
|
|
eval_id = await _create_executing_eval(client)
|
|
session = await _create_session(client, eval_id)
|
|
resp = await client.post(
|
|
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages",
|
|
json={"content": "我要退货"},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
data = resp.json()
|
|
assert data["reply"] == "好的，已为您发起退货申请"
|
|
assert data["turn_count"] == 1
|
|
|
|
msgs = await client.get(f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages")
|
|
messages = msgs.json()["messages"]
|
|
assert len(messages) == 2
|
|
assert messages[0]["role"] == "user"
|
|
assert messages[0]["content"] == "我要退货"
|
|
assert messages[1]["role"] == "assistant"
|
|
assert messages[1]["content"] == "好的，已为您发起退货申请"
|
|
assert messages[1]["latency_ms"] is not None
|
|
|
|
detail = await client.get(f"/api/intelligent-evals/{eval_id}/sessions")
|
|
assert detail.json()["sessions"][0]["turn_count"] == 1
|
|
|
|
async def test_turn_increments_across_rounds(self, client, seeded_db, mock_channel):
|
|
eval_id = await _create_executing_eval(client)
|
|
session = await _create_session(client, eval_id)
|
|
for expected in (1, 2, 3):
|
|
resp = await client.post(
|
|
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages",
|
|
json={"content": f"第{expected}轮"},
|
|
)
|
|
assert resp.json()["turn_count"] == expected
|
|
|
|
async def test_turn_unknown_session_returns_404(self, client, seeded_db, mock_channel):
|
|
resp = await client.post("/api/intelligent-evals/nope/sessions/nope/messages", json={"content": "hi"})
|
|
assert resp.status_code == 404
|
|
|
|
async def test_channel_send_failure_returns_502(self, client, seeded_db, monkeypatch):
|
|
from agenteval.intelligent_eval import lifecycle as lifecycle_module
|
|
|
|
from tests.unit.mock_channel import MockChannel
|
|
|
|
channel = MockChannel(send_ok=False)
|
|
|
|
class _StubFactory:
|
|
@staticmethod
|
|
def create(target):
|
|
return channel
|
|
|
|
monkeypatch.setattr(lifecycle_module, "ChannelFactory", _StubFactory)
|
|
eval_id = await _create_executing_eval(client)
|
|
session = await _create_session(client, eval_id)
|
|
resp = await client.post(
|
|
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages",
|
|
json={"content": "我要退货"},
|
|
)
|
|
assert resp.status_code == 502
|
|
|
|
|
|
class TestCloseSession:
|
|
async def test_close_records_verdict(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
session = await _create_session(client, eval_id)
|
|
verdict = {"passed": True, "score": 0.85, "notes": "退货流程顺畅"}
|
|
resp = await client.post(
|
|
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close",
|
|
json={"verdict": verdict},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
data = resp.json()
|
|
assert data["status"] == "completed"
|
|
assert data["verdict"] == verdict
|
|
assert data["closed_at"] is not None
|
|
|
|
eval_resp = await client.get(f"/api/intelligent-evals/{eval_id}")
|
|
assert eval_resp.json()["completed_sessions"] == 1
|
|
|
|
async def test_double_close_returns_409(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
session = await _create_session(client, eval_id)
|
|
await client.post(
|
|
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close",
|
|
json={"verdict": {"passed": True}},
|
|
)
|
|
resp = await client.post(
|
|
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close",
|
|
json={"verdict": {"passed": True}},
|
|
)
|
|
assert resp.status_code == 409
|
|
|
|
async def test_message_after_close_returns_409(self, client, seeded_db, monkeypatch):
|
|
from agenteval.intelligent_eval import lifecycle as lifecycle_module
|
|
|
|
from tests.unit.mock_channel import MockChannel
|
|
|
|
class _StubFactory:
|
|
@staticmethod
|
|
def create(target):
|
|
return MockChannel()
|
|
|
|
monkeypatch.setattr(lifecycle_module, "ChannelFactory", _StubFactory)
|
|
eval_id = await _create_executing_eval(client)
|
|
session = await _create_session(client, eval_id)
|
|
await client.post(
|
|
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close",
|
|
json={"verdict": {"passed": True}},
|
|
)
|
|
resp = await client.post(
|
|
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages",
|
|
json={"content": "还在吗"},
|
|
)
|
|
assert resp.status_code == 409
|
|
|
|
async def test_close_unknown_session_returns_404(self, client, seeded_db):
|
|
resp = await client.post(
|
|
"/api/intelligent-evals/nope/sessions/nope/close",
|
|
json={"verdict": {"passed": True}},
|
|
)
|
|
assert resp.status_code == 404
|
|
|
|
|
|
class TestListSessions:
|
|
async def test_list_sessions_unknown_eval_returns_404(self, client, seeded_db):
|
|
resp = await client.get("/api/intelligent-evals/nope/sessions")
|
|
assert resp.status_code == 404
|
|
|
|
async def test_messages_unknown_session_returns_404(self, client, seeded_db):
|
|
resp = await client.get("/api/intelligent-evals/nope/sessions/nope/messages")
|
|
assert resp.status_code == 404
|
|
|
|
async def test_session_accessed_via_wrong_eval_returns_404(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
session = await _create_session(client, eval_id)
|
|
other = await _create_eval(client, name="other")
|
|
resp = await client.get(f"/api/intelligent-evals/{other['id']}/sessions/{session['id']}/messages")
|
|
assert resp.status_code == 404
|
|
|
|
async def test_session_mutation_via_wrong_eval_returns_404(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
session = await _create_session(client, eval_id)
|
|
other = await _create_executing_eval(client)
|
|
|
|
message = await client.post(
|
|
f"/api/intelligent-evals/{other}/sessions/{session['id']}/messages",
|
|
json={"content": "越权消息"},
|
|
)
|
|
assert message.status_code == 404
|
|
|
|
closed = await client.post(
|
|
f"/api/intelligent-evals/{other}/sessions/{session['id']}/close",
|
|
json={"verdict": {"passed": False}},
|
|
)
|
|
assert closed.status_code == 404
|
|
|
|
|
|
def _report_payload() -> dict:
|
|
return {
|
|
"summary": "整体表现良好。",
|
|
"scores": {"退货流程": 0.7},
|
|
"findings": [
|
|
{
|
|
"issue": "未主动确认订单号",
|
|
"severity": "high",
|
|
"dimension": "退货流程",
|
|
"evidence": [
|
|
{
|
|
"session_id": "s-1",
|
|
"turn_index": 3,
|
|
"user_said": "我要退货",
|
|
"assistant_replied": "好的",
|
|
}
|
|
],
|
|
"suggestion": "增加确认步骤",
|
|
}
|
|
],
|
|
"highlights": [{"description": "上下文连贯", "dimension": "多轮追问"}],
|
|
"priority_recommendations": ["先修退货确认"],
|
|
}
|
|
|
|
|
|
class TestReport:
|
|
async def test_submit_report_completes_eval(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
|
assert resp.status_code == 200, resp.text
|
|
data = resp.json()
|
|
assert data["status"] == "completed"
|
|
assert data["completed_at"] is not None
|
|
assert data["report"]["summary"] == "整体表现良好。"
|
|
|
|
async def test_get_report_round_trip(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
|
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report")
|
|
assert resp.status_code == 200
|
|
report = resp.json()
|
|
assert report["summary"] == "整体表现良好。"
|
|
assert report["findings"][0]["issue"] == "未主动确认订单号"
|
|
|
|
async def test_markdown_export(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
|
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown")
|
|
assert resp.status_code == 200
|
|
assert resp.headers["content-type"].startswith("text/markdown")
|
|
assert "# 客服助手智能评估" in resp.text
|
|
assert "未主动确认订单号" in resp.text
|
|
|
|
async def test_submit_report_requires_executing(self, client, seeded_db):
|
|
ev = await _create_eval(client) # planning state
|
|
resp = await client.put(f"/api/intelligent-evals/{ev['id']}/report", json={"report": _report_payload()})
|
|
assert resp.status_code == 409
|
|
|
|
async def test_submit_report_rejects_invalid_structure(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
bad = {"summary": "缺 findings"}
|
|
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": bad})
|
|
assert resp.status_code == 422
|
|
|
|
async def test_submit_report_rejects_finding_missing_fields(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
payload = _report_payload()
|
|
payload["findings"] = [{"issue": "只有 issue"}]
|
|
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": payload})
|
|
assert resp.status_code == 422
|
|
|
|
async def test_get_report_before_submission_returns_404(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report")
|
|
assert resp.status_code == 404
|
|
|
|
async def test_markdown_before_submission_returns_404(self, client, seeded_db):
|
|
eval_id = await _create_executing_eval(client)
|
|
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown")
|
|
assert resp.status_code == 404
|