eb4944a8bd
常见故障自愈有上限，超限收敛终态且可见：任务 attempts 上限、会话过期、 planning 双闸、executing 超窗兜底、触发失败计数判死、孤儿 agent 双管、 fire-and-forget 触发；open_session 预算硬闸门、settle 按终态区分、报告 scores 归一化；cron 池遗留面全删。
601 lines
25 KiB
Python
601 lines
25 KiB
Python
"""Integration tests for intelligent eval lifecycle + session + report API (tickets 02, 03, 04).
|
||
|
||
Uses httpx.AsyncClient with app= to drive the FastAPI app in-process.
|
||
Covers the full lifecycle (create → plan → approve → execute → cancel),
|
||
reject/resubmit flow, illegal transition rejections (409), the session
|
||
lifecycle (create → message → close) with channel I/O stubbed via MockChannel,
|
||
and the report flow (submit → completed → read → Markdown export).
|
||
"""
|
||
|
||
import pytest
|
||
from agenteval.models import ChannelType, EvalTarget, PlatformType, TargetStatus
|
||
from agenteval.storage.repository import TargetRepository
|
||
from agenteval.web.app import app
|
||
from httpx import ASGITransport, AsyncClient
|
||
|
||
|
||
@pytest.fixture()
|
||
def seeded_db(db_session, monkeypatch):
|
||
"""Patch get_session/get_db to the test session and seed a target."""
|
||
from agenteval.storage import db as db_module
|
||
from agenteval.storage import repository as repo_module
|
||
from agenteval.web import app as app_module
|
||
|
||
monkeypatch.setattr(app_module, "init_db", lambda: None)
|
||
|
||
def _test_get_session():
|
||
return db_session
|
||
|
||
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
||
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
||
|
||
from agenteval.web.deps import get_db
|
||
|
||
def _test_get_db():
|
||
try:
|
||
yield db_session
|
||
finally:
|
||
pass
|
||
|
||
app.dependency_overrides[get_db] = _test_get_db
|
||
|
||
target = EvalTarget(
|
||
id="t-1",
|
||
name="mock-target",
|
||
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
||
channel_type=ChannelType.TUTU_API,
|
||
channel_config={"base_url": "http://mock", "token": "x"},
|
||
status=TargetStatus.ACTIVE,
|
||
)
|
||
TargetRepository(db_session).create(target)
|
||
|
||
yield db_session
|
||
app.dependency_overrides.clear()
|
||
|
||
|
||
@pytest.fixture()
|
||
async def client():
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
async def _create_eval(client, **overrides) -> dict:
|
||
payload = {
|
||
"name": "客服助手智能评估",
|
||
"target_id": "t-1",
|
||
"goal": "评估退货流程处理能力",
|
||
"seeds": {"personas": ["急躁老客户"], "goals": ["完成退货"]},
|
||
"intent": "考察退货全流程",
|
||
"role_description": "模拟真实用户",
|
||
"time_window_hours": 24,
|
||
}
|
||
payload.update(overrides)
|
||
resp = await client.post("/api/intelligent-evals", json=payload)
|
||
assert resp.status_code == 200, resp.text
|
||
return resp.json()
|
||
|
||
|
||
async def _submit_plan(client, eval_id: str, plan: dict | None = None) -> dict:
|
||
if plan is None:
|
||
plan = {
|
||
"dimensions": ["退货流程", "投诉处理"],
|
||
"virtual_users": [{"persona": {"background": "老客户"}, "goal": "完成退货"}],
|
||
"time_distribution": [{"time_slot": "0-2h", "sessions": 1, "scenario": "早间咨询"}],
|
||
"estimated_sessions": 3,
|
||
"budget": {"max_turns_per_session": 12, "total_max_turns": 36},
|
||
"completion_criteria": "每个维度至少一个会话",
|
||
}
|
||
resp = await client.put(f"/api/intelligent-evals/{eval_id}/plan", json={"plan": plan})
|
||
assert resp.status_code == 200, resp.text
|
||
return resp.json()
|
||
|
||
|
||
class TestCreateEval:
|
||
async def test_create_transitions_to_planning(self, client, seeded_db):
|
||
data = await _create_eval(client)
|
||
assert data["status"] == "planning"
|
||
assert data["name"] == "客服助手智能评估"
|
||
assert data["goal"] == "评估退货流程处理能力"
|
||
assert data["time_window_hours"] == 24
|
||
assert "id" in data
|
||
|
||
async def test_create_requires_name(self, client, seeded_db):
|
||
resp = await client.post(
|
||
"/api/intelligent-evals",
|
||
json={
|
||
"name": "",
|
||
"target_id": "t-1",
|
||
"goal": "test",
|
||
},
|
||
)
|
||
assert resp.status_code == 422
|
||
|
||
async def test_create_requires_goal(self, client, seeded_db):
|
||
resp = await client.post(
|
||
"/api/intelligent-evals",
|
||
json={
|
||
"name": "test",
|
||
"target_id": "t-1",
|
||
"goal": "",
|
||
},
|
||
)
|
||
assert resp.status_code == 422
|
||
|
||
async def test_create_requires_existing_target(self, client, seeded_db):
|
||
resp = await client.post(
|
||
"/api/intelligent-evals",
|
||
json={"name": "test", "target_id": "missing", "goal": "evaluate"},
|
||
)
|
||
assert resp.status_code == 404
|
||
|
||
|
||
class TestPlanApproval:
|
||
async def test_submit_plan_transitions_to_pending_approval(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
data = await _submit_plan(client, ev["id"])
|
||
assert data["status"] == "pending_approval"
|
||
assert data["plan"]["dimensions"] == ["退货流程", "投诉处理"]
|
||
|
||
async def test_approve_transitions_to_executing(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
await _submit_plan(client, ev["id"])
|
||
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["status"] == "executing"
|
||
assert data["started_at"] is not None
|
||
|
||
async def test_reject_transitions_back_to_planning(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
await _submit_plan(client, ev["id"])
|
||
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/reject", json={"feedback": "缺少投诉维度"})
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["status"] == "planning"
|
||
assert data["plan_feedback"] == "缺少投诉维度"
|
||
|
||
async def test_reject_then_resubmit(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
await _submit_plan(client, ev["id"], plan={"dimensions": ["A"]})
|
||
await client.post(f"/api/intelligent-evals/{ev['id']}/reject", json={"feedback": "加B"})
|
||
data = await _submit_plan(client, ev["id"], plan={"dimensions": ["A", "B"]})
|
||
assert data["status"] == "pending_approval"
|
||
assert data["plan"]["dimensions"] == ["A", "B"]
|
||
assert data["plan_feedback"] is None
|
||
|
||
|
||
class TestCancel:
|
||
async def _create_executing(self, client) -> str:
|
||
ev = await _create_eval(client)
|
||
await _submit_plan(client, ev["id"])
|
||
await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
||
return ev["id"]
|
||
|
||
async def test_cancel_executing(self, client, seeded_db):
|
||
eval_id = await self._create_executing(client)
|
||
resp = await client.post(f"/api/intelligent-evals/{eval_id}/cancel")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["status"] == "cancelled"
|
||
assert data["completed_at"] is not None
|
||
|
||
async def test_cancel_pending_approval(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
await _submit_plan(client, ev["id"])
|
||
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/cancel")
|
||
assert resp.status_code == 200
|
||
assert resp.json()["status"] == "cancelled"
|
||
|
||
|
||
class TestIllegalTransitions:
|
||
async def test_approve_from_planning_returns_409(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
||
assert resp.status_code == 409
|
||
|
||
async def test_double_cancel_returns_409(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
await _submit_plan(client, ev["id"])
|
||
await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
||
await client.post(f"/api/intelligent-evals/{ev['id']}/cancel")
|
||
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/cancel")
|
||
assert resp.status_code == 409
|
||
|
||
async def test_submit_plan_from_executing_returns_409(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
await _submit_plan(client, ev["id"])
|
||
await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
||
resp = await client.put(f"/api/intelligent-evals/{ev['id']}/plan", json={"plan": {"new": True}})
|
||
assert resp.status_code == 409
|
||
|
||
async def test_not_found_returns_404(self, client, seeded_db):
|
||
resp = await client.get("/api/intelligent-evals/nonexistent")
|
||
assert resp.status_code == 404
|
||
|
||
|
||
class TestListAndGet:
|
||
async def test_list_empty(self, client, seeded_db):
|
||
resp = await client.get("/api/intelligent-evals")
|
||
assert resp.status_code == 200
|
||
assert resp.json()["intelligent_evals"] == []
|
||
|
||
async def test_list_returns_created(self, client, seeded_db):
|
||
await _create_eval(client, name="e1")
|
||
await _create_eval(client, name="e2")
|
||
resp = await client.get("/api/intelligent-evals")
|
||
assert len(resp.json()["intelligent_evals"]) == 2
|
||
|
||
async def test_get_by_id(self, client, seeded_db):
|
||
ev = await _create_eval(client, name="e1")
|
||
resp = await client.get(f"/api/intelligent-evals/{ev['id']}")
|
||
assert resp.status_code == 200
|
||
assert resp.json()["name"] == "e1"
|
||
|
||
|
||
async def _create_executing_eval(client) -> str:
|
||
ev = await _create_eval(client)
|
||
await _submit_plan(client, ev["id"])
|
||
await client.post(f"/api/intelligent-evals/{ev['id']}/approve")
|
||
return ev["id"]
|
||
|
||
|
||
async def _create_session(client, eval_id: str, **overrides) -> dict:
|
||
payload = {
|
||
"persona": {"background": "急躁老客户", "style": "直接"},
|
||
"goal": "完成退货",
|
||
"dimension": "退货流程",
|
||
}
|
||
payload.update(overrides)
|
||
resp = await client.post(f"/api/intelligent-evals/{eval_id}/sessions", json=payload)
|
||
assert resp.status_code == 200, resp.text
|
||
return resp.json()
|
||
|
||
|
||
class TestCreateSession:
|
||
async def test_create_session_in_executing(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
data = await _create_session(client, eval_id)
|
||
assert data["eval_id"] == eval_id
|
||
assert data["target_id"] == "t-1"
|
||
assert data["status"] == "running"
|
||
assert data["persona"]["background"] == "急躁老客户"
|
||
assert data["dimension"] == "退货流程"
|
||
assert data["turn_count"] == 0
|
||
|
||
async def test_create_session_in_planning_returns_409(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
resp = await client.post(f"/api/intelligent-evals/{ev['id']}/sessions", json={"goal": "g"})
|
||
assert resp.status_code == 409
|
||
|
||
async def test_create_session_requires_goal(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
resp = await client.post(f"/api/intelligent-evals/{eval_id}/sessions", json={"goal": ""})
|
||
assert resp.status_code == 422
|
||
|
||
async def test_create_session_unknown_eval_returns_404(self, client, seeded_db):
|
||
resp = await client.post("/api/intelligent-evals/nope/sessions", json={"goal": "g"})
|
||
assert resp.status_code == 404
|
||
|
||
async def test_eval_response_counts_sessions(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
await _create_session(client, eval_id)
|
||
resp = await client.get(f"/api/intelligent-evals/{eval_id}")
|
||
data = resp.json()
|
||
assert data["session_count"] == 1
|
||
assert data["completed_sessions"] == 0
|
||
assert len(data["sessions"]) == 1
|
||
assert "messages" not in data["sessions"][0]
|
||
|
||
|
||
class TestConductTurn:
|
||
@pytest.fixture()
|
||
def mock_channel(self, monkeypatch):
|
||
from agenteval.intelligent_eval import lifecycle as lifecycle_module
|
||
|
||
from tests.unit.mock_channel import MockChannel
|
||
|
||
channel = MockChannel(reply_text="好的，已为您发起退货申请")
|
||
|
||
class _StubFactory:
|
||
@staticmethod
|
||
def create(target):
|
||
return channel
|
||
|
||
monkeypatch.setattr(lifecycle_module, "ChannelFactory", _StubFactory)
|
||
return channel
|
||
|
||
async def test_turn_round_trip_persists_messages(self, client, seeded_db, mock_channel):
|
||
eval_id = await _create_executing_eval(client)
|
||
session = await _create_session(client, eval_id)
|
||
resp = await client.post(
|
||
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages",
|
||
json={"content": "我要退货"},
|
||
)
|
||
assert resp.status_code == 200, resp.text
|
||
data = resp.json()
|
||
assert data["reply"] == "好的，已为您发起退货申请"
|
||
assert data["turn_count"] == 1
|
||
|
||
msgs = await client.get(f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages")
|
||
messages = msgs.json()["messages"]
|
||
assert len(messages) == 2
|
||
assert messages[0]["role"] == "user"
|
||
assert messages[0]["content"] == "我要退货"
|
||
assert messages[1]["role"] == "assistant"
|
||
assert messages[1]["content"] == "好的，已为您发起退货申请"
|
||
assert messages[1]["latency_ms"] is not None
|
||
|
||
detail = await client.get(f"/api/intelligent-evals/{eval_id}/sessions")
|
||
assert detail.json()["sessions"][0]["turn_count"] == 1
|
||
|
||
async def test_turn_increments_across_rounds(self, client, seeded_db, mock_channel):
|
||
eval_id = await _create_executing_eval(client)
|
||
session = await _create_session(client, eval_id)
|
||
for expected in (1, 2, 3):
|
||
resp = await client.post(
|
||
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages",
|
||
json={"content": f"第{expected}轮"},
|
||
)
|
||
assert resp.json()["turn_count"] == expected
|
||
|
||
async def test_turn_unknown_session_returns_404(self, client, seeded_db, mock_channel):
|
||
resp = await client.post("/api/intelligent-evals/nope/sessions/nope/messages", json={"content": "hi"})
|
||
assert resp.status_code == 404
|
||
|
||
async def test_channel_send_failure_returns_502(self, client, seeded_db, monkeypatch):
|
||
from agenteval.intelligent_eval import lifecycle as lifecycle_module
|
||
|
||
from tests.unit.mock_channel import MockChannel
|
||
|
||
channel = MockChannel(send_ok=False)
|
||
|
||
class _StubFactory:
|
||
@staticmethod
|
||
def create(target):
|
||
return channel
|
||
|
||
monkeypatch.setattr(lifecycle_module, "ChannelFactory", _StubFactory)
|
||
eval_id = await _create_executing_eval(client)
|
||
session = await _create_session(client, eval_id)
|
||
resp = await client.post(
|
||
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages",
|
||
json={"content": "我要退货"},
|
||
)
|
||
assert resp.status_code == 502
|
||
|
||
|
||
class TestCloseSession:
|
||
async def test_close_records_verdict(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
session = await _create_session(client, eval_id)
|
||
verdict = {"passed": True, "score": 0.85, "notes": "退货流程顺畅"}
|
||
resp = await client.post(
|
||
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close",
|
||
json={"verdict": verdict},
|
||
)
|
||
assert resp.status_code == 200, resp.text
|
||
data = resp.json()
|
||
assert data["status"] == "completed"
|
||
assert data["verdict"] == verdict
|
||
assert data["closed_at"] is not None
|
||
|
||
eval_resp = await client.get(f"/api/intelligent-evals/{eval_id}")
|
||
assert eval_resp.json()["completed_sessions"] == 1
|
||
|
||
async def test_double_close_returns_409(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
session = await _create_session(client, eval_id)
|
||
await client.post(
|
||
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close",
|
||
json={"verdict": {"passed": True}},
|
||
)
|
||
resp = await client.post(
|
||
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close",
|
||
json={"verdict": {"passed": True}},
|
||
)
|
||
assert resp.status_code == 409
|
||
|
||
async def test_message_after_close_returns_409(self, client, seeded_db, monkeypatch):
|
||
from agenteval.intelligent_eval import lifecycle as lifecycle_module
|
||
|
||
from tests.unit.mock_channel import MockChannel
|
||
|
||
class _StubFactory:
|
||
@staticmethod
|
||
def create(target):
|
||
return MockChannel()
|
||
|
||
monkeypatch.setattr(lifecycle_module, "ChannelFactory", _StubFactory)
|
||
eval_id = await _create_executing_eval(client)
|
||
session = await _create_session(client, eval_id)
|
||
await client.post(
|
||
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close",
|
||
json={"verdict": {"passed": True}},
|
||
)
|
||
resp = await client.post(
|
||
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages",
|
||
json={"content": "还在吗"},
|
||
)
|
||
assert resp.status_code == 409
|
||
|
||
async def test_close_unknown_session_returns_404(self, client, seeded_db):
|
||
resp = await client.post(
|
||
"/api/intelligent-evals/nope/sessions/nope/close",
|
||
json={"verdict": {"passed": True}},
|
||
)
|
||
assert resp.status_code == 404
|
||
|
||
|
||
class TestListSessions:
|
||
async def test_list_sessions_unknown_eval_returns_404(self, client, seeded_db):
|
||
resp = await client.get("/api/intelligent-evals/nope/sessions")
|
||
assert resp.status_code == 404
|
||
|
||
async def test_messages_unknown_session_returns_404(self, client, seeded_db):
|
||
resp = await client.get("/api/intelligent-evals/nope/sessions/nope/messages")
|
||
assert resp.status_code == 404
|
||
|
||
async def test_session_accessed_via_wrong_eval_returns_404(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
session = await _create_session(client, eval_id)
|
||
other = await _create_eval(client, name="other")
|
||
resp = await client.get(f"/api/intelligent-evals/{other['id']}/sessions/{session['id']}/messages")
|
||
assert resp.status_code == 404
|
||
|
||
async def test_session_mutation_via_wrong_eval_returns_404(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
session = await _create_session(client, eval_id)
|
||
other = await _create_executing_eval(client)
|
||
|
||
message = await client.post(
|
||
f"/api/intelligent-evals/{other}/sessions/{session['id']}/messages",
|
||
json={"content": "越权消息"},
|
||
)
|
||
assert message.status_code == 404
|
||
|
||
closed = await client.post(
|
||
f"/api/intelligent-evals/{other}/sessions/{session['id']}/close",
|
||
json={"verdict": {"passed": False}},
|
||
)
|
||
assert closed.status_code == 404
|
||
|
||
|
||
def _report_payload() -> dict:
|
||
return {
|
||
"summary": "整体表现良好。",
|
||
"scores": {"退货流程": 0.7},
|
||
"findings": [
|
||
{
|
||
"issue": "未主动确认订单号",
|
||
"severity": "high",
|
||
"dimension": "退货流程",
|
||
"evidence": [
|
||
{
|
||
"session_id": "s-1",
|
||
"turn_index": 3,
|
||
"user_said": "我要退货",
|
||
"assistant_replied": "好的",
|
||
}
|
||
],
|
||
"suggestion": "增加确认步骤",
|
||
}
|
||
],
|
||
"highlights": [{"description": "上下文连贯", "dimension": "多轮追问"}],
|
||
"priority_recommendations": ["先修退货确认"],
|
||
}
|
||
|
||
|
||
class TestReport:
|
||
async def test_submit_report_completes_eval(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
||
assert resp.status_code == 200, resp.text
|
||
data = resp.json()
|
||
assert data["status"] == "completed"
|
||
assert data["completed_at"] is not None
|
||
assert data["report"]["summary"] == "整体表现良好。"
|
||
|
||
async def test_get_report_round_trip(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
||
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report")
|
||
assert resp.status_code == 200
|
||
report = resp.json()
|
||
assert report["summary"] == "整体表现良好。"
|
||
assert report["findings"][0]["issue"] == "未主动确认订单号"
|
||
|
||
async def test_markdown_export(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
||
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown")
|
||
assert resp.status_code == 200
|
||
assert resp.headers["content-type"].startswith("text/markdown")
|
||
assert "# 客服助手智能评估" in resp.text
|
||
assert "未主动确认订单号" in resp.text
|
||
|
||
async def test_submit_report_requires_executing(self, client, seeded_db):
|
||
ev = await _create_eval(client) # planning state
|
||
resp = await client.put(f"/api/intelligent-evals/{ev['id']}/report", json={"report": _report_payload()})
|
||
assert resp.status_code == 409
|
||
|
||
async def test_submit_report_rejects_invalid_structure(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
bad = {"summary": "缺 findings"}
|
||
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": bad})
|
||
assert resp.status_code == 422
|
||
|
||
async def test_submit_report_rejects_finding_missing_fields(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
payload = _report_payload()
|
||
payload["findings"] = [{"issue": "只有 issue"}]
|
||
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": payload})
|
||
assert resp.status_code == 422
|
||
|
||
async def test_get_report_before_submission_returns_404(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report")
|
||
assert resp.status_code == 404
|
||
|
||
async def test_submit_report_rejects_when_sessions_not_closed(self, client, seeded_db):
|
||
"""评估不应在会话未全部 close 时被标 completed（否则进度 0% 但状态完成）。
|
||
|
||
复现 t480 真实问题：worker 建会话但未 close，analyst 仍提交报告 →
|
||
submit_report 无条件 executing→completed，completed_sessions=0。
|
||
"""
|
||
eval_id = await _create_executing_eval(client)
|
||
await _create_session(client, eval_id) # 建 1 个 running 会话，不 close
|
||
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
||
# 会话未全部 close → 应拒绝（409），而非 200 直接 completed
|
||
assert resp.status_code == 409, resp.text
|
||
|
||
async def test_submit_report_allows_when_sessions_closed(self, client, seeded_db):
|
||
"""会话全部 close 后提交报告应成功（正常 analyst 路径不被破坏）。"""
|
||
eval_id = await _create_executing_eval(client)
|
||
session = await _create_session(client, eval_id)
|
||
resp = await client.post(
|
||
f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close",
|
||
json={"verdict": {"goal_achieved": True, "summary": "完成", "severity": "low"}},
|
||
)
|
||
assert resp.status_code == 200, resp.text
|
||
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
||
assert resp.status_code == 200, resp.text
|
||
assert resp.json()["status"] == "completed"
|
||
|
||
async def test_markdown_before_submission_returns_404(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown")
|
||
assert resp.status_code == 404
|
||
|
||
|
||
class TestExecutionProgress:
|
||
async def test_planning_stage_progress(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
|
||
resp = await client.get(f"/api/intelligent-evals/{ev['id']}/execution-progress")
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
data = resp.json()
|
||
assert data["current_stage"] == "planning"
|
||
assert data["abnormal_outcome"] is None
|
||
assert data["blocker"] is None
|
||
assert data["slots"] == []
|
||
|
||
async def test_pending_approval_marks_human_blocker(self, client, seeded_db):
|
||
ev = await _create_eval(client)
|
||
await client.put(
|
||
f"/api/intelligent-evals/{ev['id']}/plan",
|
||
json={"plan": {"time_distribution": [{"time_slot": "0-1h", "sessions": 1}]}},
|
||
)
|
||
|
||
resp = await client.get(f"/api/intelligent-evals/{ev['id']}/execution-progress")
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
data = resp.json()
|
||
assert data["current_stage"] == "approval"
|
||
assert "审批" in data["blocker"]
|
||
|
||
async def test_unknown_eval_returns_404(self, client, seeded_db):
|
||
resp = await client.get("/api/intelligent-evals/missing/execution-progress")
|
||
assert resp.status_code == 404
|