71543f042a
将「已删即 404」语义收进 IntelligentEvalRepository 单一接缝，消除三处独立裁决； 任务监控开始隐藏已删评估的任务（本 Phase 唯一刻意行为变化）。 - repository.py 新增 visible() 谓词与 require_live_eval() 服务接缝； 六处裸谓词统一走它，get()/get_including_deleted() 语义不变。 - decision_logs.py 删除本地 _require_eval，三处调用迁至 repository 接缝； count_decisions 由 len(.all()) 改为 func.count。 - task_queue.py list_tasks 与 stats 过滤已删评估的任务（行为变化）。 - web/routers/intelligent_evals.py: _require_eval_exists → _require_live_eval， 把 LookupError 翻译为 404；expired 会话 Markdown 标注下沉至 read_model.report_markdown_by_eval；配置快照 11 字段序列化收至 config_snapshot.snapshot_to_dict 单一出口。 - AGENTS.md 登记可见性纪律（已知陷阱 #6）。 - 补 characterization 测试锁定四处契约；更新 task_queue 测试以使用 真实 eval_id（可见性过滤后字面 eval_id 不再可见）。
205 lines
8.5 KiB
Python
205 lines
8.5 KiB
Python
"""Characterization tests: intelligent-eval visibility semantics (Phase 1).
|
||
|
||
锁定逻辑删除可见性收敛（require_live_eval 接缝）后的行为契约。覆盖四处：
|
||
|
||
1. 「已删即 404」的统一裁决：config-snapshots 详情/对比端点与
|
||
decision_logs 服务层都经 require_live_eval 接缝。
|
||
2. 任务队列监控**隐藏**已删评估的任务（Phase 1.6 唯一的刻意行为变化，
|
||
锁定变更后的新契约）。
|
||
3. config-snapshots 端点的响应形状（11 字段序列化，snapshot_to_dict 单一出口）。
|
||
4. expired 会话在 Markdown 报告中的「不完整证据」标注文本（ADR-0011）。
|
||
"""
|
||
|
||
import pytest
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import (
|
||
IntelligentEvalSessionDB,
|
||
IntelligentEvalTaskQueueDB,
|
||
utc_now,
|
||
)
|
||
from sqlmodel import select
|
||
|
||
from .test_intelligent_evals_api import (
|
||
_create_eval,
|
||
_create_executing_eval,
|
||
_create_session,
|
||
_report_payload,
|
||
_submit_plan,
|
||
)
|
||
|
||
|
||
async def _complete_eval(client, eval_id: str) -> None:
|
||
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
||
assert resp.status_code == 200, resp.text
|
||
|
||
|
||
class TestSnapshotEndpointDeletedSemantics:
|
||
"""路由 _require_eval_exists 路径：config-snapshots 详情与对比。"""
|
||
|
||
async def test_deleted_eval_snapshot_detail_returns_404(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
await _complete_eval(client, eval_id)
|
||
snapshots = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"]
|
||
assert snapshots
|
||
await client.delete(f"/api/intelligent-evals/{eval_id}")
|
||
|
||
resp = await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots/{snapshots[0]['id']}")
|
||
assert resp.status_code == 404
|
||
|
||
async def test_deleted_eval_snapshot_compare_returns_404(self, client, seeded_db):
|
||
eval_id = (await _create_eval(client))["id"]
|
||
await _submit_plan(client, eval_id) # second snapshot (plan_submitted)
|
||
await client.post(f"/api/intelligent-evals/{eval_id}/approve")
|
||
await _complete_eval(client, eval_id)
|
||
snapshots = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"]
|
||
assert len(snapshots) >= 2
|
||
await client.delete(f"/api/intelligent-evals/{eval_id}")
|
||
|
||
resp = await client.post(
|
||
f"/api/intelligent-evals/{eval_id}/config-snapshots/compare",
|
||
json={"snapshot_id_1": snapshots[0]["id"], "snapshot_id_2": snapshots[1]["id"]},
|
||
)
|
||
assert resp.status_code == 404
|
||
|
||
|
||
class TestDecisionLogServiceDeletedSemantics:
|
||
"""decision_logs 经 require_live_eval 接缝：已删评估一律 LookupError。"""
|
||
|
||
async def test_deleted_eval_service_calls_raise_lookup_error(self, client, seeded_db):
|
||
from agenteval.intelligent_eval import decision_logs
|
||
|
||
eval_id = await _create_executing_eval(client)
|
||
await _complete_eval(client, eval_id)
|
||
await client.delete(f"/api/intelligent-evals/{eval_id}")
|
||
|
||
with pytest.raises(LookupError):
|
||
decision_logs.list_decision_logs(eval_id, seeded_db)
|
||
with pytest.raises(LookupError):
|
||
decision_logs.create_decision_log(eval_id, "wait", "late report", "w-1", {}, seeded_db)
|
||
with pytest.raises(LookupError):
|
||
decision_logs.append_decision_log(eval_id, "wait", "late bookkeeping", "platform", {}, seeded_db)
|
||
|
||
async def test_repo_get_hides_deleted_but_including_deleted_returns_it(self, client, seeded_db):
|
||
from agenteval.intelligent_eval.repository import IntelligentEvalRepository
|
||
|
||
eval_id = await _create_executing_eval(client)
|
||
await _complete_eval(client, eval_id)
|
||
await client.delete(f"/api/intelligent-evals/{eval_id}")
|
||
|
||
repo = IntelligentEvalRepository(seeded_db)
|
||
assert repo.get(eval_id) is None
|
||
hidden = repo.get_including_deleted(eval_id)
|
||
assert hidden is not None
|
||
assert hidden.status is IntelligentEvalStatus.DELETED
|
||
|
||
|
||
class TestTaskMonitorHidesDeletedEvals:
|
||
"""Phase 1.6 刻意行为变化：任务监控不再泄漏已删评估的任务。"""
|
||
|
||
async def test_list_tasks_hides_tasks_of_deleted_eval(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
seeded_db.add(
|
||
IntelligentEvalTaskQueueDB(
|
||
eval_id=eval_id,
|
||
status="completed",
|
||
priority=1,
|
||
reason="slot_due",
|
||
completed_at=utc_now(),
|
||
)
|
||
)
|
||
seeded_db.commit()
|
||
|
||
await _complete_eval(client, eval_id)
|
||
await client.delete(f"/api/intelligent-evals/{eval_id}")
|
||
|
||
resp = await client.get("/api/intelligent-evals/tasks")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
|
||
assert [t for t in data["tasks"] if t["eval_id"] == eval_id] == []
|
||
assert data["stats"]["completed"] == 0
|
||
|
||
by_eval = await client.get("/api/intelligent-evals/tasks", params={"eval_id": eval_id})
|
||
assert by_eval.json()["tasks"] == []
|
||
|
||
|
||
class TestSnapshotResponseShape:
|
||
"""锁定 config-snapshots 列表/详情的 11 字段序列化形状。"""
|
||
|
||
SNAPSHOT_KEYS = {
|
||
"id",
|
||
"eval_id",
|
||
"snapshot_type",
|
||
"goal",
|
||
"seeds",
|
||
"intent",
|
||
"role_description",
|
||
"time_window_hours",
|
||
"plan",
|
||
"created_at",
|
||
"created_by",
|
||
}
|
||
|
||
async def test_list_shape_and_snapshot_types(self, client, seeded_db):
|
||
eval_id = (await _create_eval(client))["id"]
|
||
await _submit_plan(client, eval_id)
|
||
|
||
snapshots = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"]
|
||
assert len(snapshots) == 2
|
||
assert {s["snapshot_type"] for s in snapshots} == {"created", "plan_submitted"}
|
||
for s in snapshots:
|
||
assert set(s.keys()) == self.SNAPSHOT_KEYS
|
||
assert s["eval_id"] == eval_id
|
||
created = next(s for s in snapshots if s["snapshot_type"] == "created")
|
||
assert created["created_by"] == "user"
|
||
assert created["goal"] == "评估退货流程处理能力"
|
||
assert created["time_window_hours"] == 24
|
||
assert created["plan"] is None
|
||
|
||
async def test_detail_shape_matches_list_item(self, client, seeded_db):
|
||
eval_id = (await _create_eval(client))["id"]
|
||
listed = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"][0]
|
||
|
||
detail = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots/{listed['id']}")).json()
|
||
assert set(detail.keys()) == self.SNAPSHOT_KEYS
|
||
assert detail == listed
|
||
|
||
async def test_snapshot_of_other_eval_returns_404(self, client, seeded_db):
|
||
eval_id = (await _create_eval(client))["id"]
|
||
other_id = (await _create_eval(client, name="另一个评估"))["id"]
|
||
foreign = (await client.get(f"/api/intelligent-evals/{other_id}/config-snapshots")).json()["snapshots"][0]
|
||
|
||
resp = await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots/{foreign['id']}")
|
||
assert resp.status_code == 404
|
||
|
||
|
||
class TestExpiredSessionMarkdownAnnotation:
|
||
"""ADR-0011：expired 会话在 Markdown 报告尾部标注不完整证据。"""
|
||
|
||
async def test_markdown_annotates_expired_sessions(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
session = await _create_session(client, eval_id)
|
||
|
||
row = seeded_db.exec(select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.id == session["id"])).one()
|
||
row.status = "expired"
|
||
row.turn_count = 3
|
||
row.closed_at = utc_now()
|
||
seeded_db.add(row)
|
||
seeded_db.commit()
|
||
|
||
await _complete_eval(client, eval_id)
|
||
|
||
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown")
|
||
assert resp.status_code == 200
|
||
text = resp.text
|
||
assert "## 不完整证据会话" in text
|
||
assert f"- 会话 `{session['id']}`（3 轮）：60 分钟无新轮次过期，证据不完整" in text
|
||
|
||
async def test_markdown_without_expired_sessions_has_no_annotation(self, client, seeded_db):
|
||
eval_id = await _create_executing_eval(client)
|
||
await _complete_eval(client, eval_id)
|
||
|
||
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown")
|
||
assert resp.status_code == 200
|
||
assert "不完整证据会话" not in resp.text
|