"""Characterization tests: intelligent-eval visibility semantics (Phase 1). 锁定逻辑删除可见性收敛(require_live_eval 接缝)后的行为契约。覆盖四处: 1. 「已删即 404」的统一裁决:config-snapshots 详情/对比端点与 decision_logs 服务层都经 require_live_eval 接缝。 2. 任务队列监控**隐藏**已删评估的任务(Phase 1.6 唯一的刻意行为变化, 锁定变更后的新契约)。 3. config-snapshots 端点的响应形状(11 字段序列化,snapshot_to_dict 单一出口)。 4. expired 会话在 Markdown 报告中的「不完整证据」标注文本(ADR-0011)。 """ import pytest from agenteval.intelligent_eval.models import IntelligentEvalStatus from agenteval.storage.db import ( IntelligentEvalSessionDB, IntelligentEvalTaskQueueDB, utc_now, ) from sqlmodel import select from .test_intelligent_evals_api import ( _create_eval, _create_executing_eval, _create_session, _report_payload, _submit_plan, ) async def _complete_eval(client, eval_id: str) -> None: resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()}) assert resp.status_code == 200, resp.text class TestSnapshotEndpointDeletedSemantics: """路由 _require_eval_exists 路径:config-snapshots 详情与对比。""" async def test_deleted_eval_snapshot_detail_returns_404(self, client, seeded_db): eval_id = await _create_executing_eval(client) await _complete_eval(client, eval_id) snapshots = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"] assert snapshots await client.delete(f"/api/intelligent-evals/{eval_id}") resp = await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots/{snapshots[0]['id']}") assert resp.status_code == 404 async def test_deleted_eval_snapshot_compare_returns_404(self, client, seeded_db): eval_id = (await _create_eval(client))["id"] await _submit_plan(client, eval_id) # second snapshot (plan_submitted) await client.post(f"/api/intelligent-evals/{eval_id}/approve") await _complete_eval(client, eval_id) snapshots = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"] assert len(snapshots) >= 2 await client.delete(f"/api/intelligent-evals/{eval_id}") resp = await client.post( f"/api/intelligent-evals/{eval_id}/config-snapshots/compare", json={"snapshot_id_1": snapshots[0]["id"], "snapshot_id_2": snapshots[1]["id"]}, ) assert resp.status_code == 404 class TestDecisionLogServiceDeletedSemantics: """decision_logs 经 require_live_eval 接缝:已删评估一律 LookupError。""" async def test_deleted_eval_service_calls_raise_lookup_error(self, client, seeded_db): from agenteval.intelligent_eval import decision_logs eval_id = await _create_executing_eval(client) await _complete_eval(client, eval_id) await client.delete(f"/api/intelligent-evals/{eval_id}") with pytest.raises(LookupError): decision_logs.list_decision_logs(eval_id, seeded_db) with pytest.raises(LookupError): decision_logs.create_decision_log(eval_id, "wait", "late report", "w-1", {}, seeded_db) with pytest.raises(LookupError): decision_logs.append_decision_log(eval_id, "wait", "late bookkeeping", "platform", {}, seeded_db) async def test_repo_get_hides_deleted_but_including_deleted_returns_it(self, client, seeded_db): from agenteval.intelligent_eval.repository import IntelligentEvalRepository eval_id = await _create_executing_eval(client) await _complete_eval(client, eval_id) await client.delete(f"/api/intelligent-evals/{eval_id}") repo = IntelligentEvalRepository(seeded_db) assert repo.get(eval_id) is None hidden = repo.get_including_deleted(eval_id) assert hidden is not None assert hidden.status is IntelligentEvalStatus.DELETED class TestTaskMonitorHidesDeletedEvals: """Phase 1.6 刻意行为变化:任务监控不再泄漏已删评估的任务。""" async def test_list_tasks_hides_tasks_of_deleted_eval(self, client, seeded_db): eval_id = await _create_executing_eval(client) seeded_db.add( IntelligentEvalTaskQueueDB( eval_id=eval_id, status="completed", priority=1, reason="slot_due", completed_at=utc_now(), ) ) seeded_db.commit() await _complete_eval(client, eval_id) await client.delete(f"/api/intelligent-evals/{eval_id}") resp = await client.get("/api/intelligent-evals/tasks") assert resp.status_code == 200 data = resp.json() assert [t for t in data["tasks"] if t["eval_id"] == eval_id] == [] assert data["stats"]["completed"] == 0 by_eval = await client.get("/api/intelligent-evals/tasks", params={"eval_id": eval_id}) assert by_eval.json()["tasks"] == [] class TestSnapshotResponseShape: """锁定 config-snapshots 列表/详情的 11 字段序列化形状。""" SNAPSHOT_KEYS = { "id", "eval_id", "snapshot_type", "goal", "seeds", "intent", "role_description", "time_window_hours", "plan", "created_at", "created_by", } async def test_list_shape_and_snapshot_types(self, client, seeded_db): eval_id = (await _create_eval(client))["id"] await _submit_plan(client, eval_id) snapshots = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"] assert len(snapshots) == 2 assert {s["snapshot_type"] for s in snapshots} == {"created", "plan_submitted"} for s in snapshots: assert set(s.keys()) == self.SNAPSHOT_KEYS assert s["eval_id"] == eval_id created = next(s for s in snapshots if s["snapshot_type"] == "created") assert created["created_by"] == "user" assert created["goal"] == "评估退货流程处理能力" assert created["time_window_hours"] == 24 assert created["plan"] is None async def test_detail_shape_matches_list_item(self, client, seeded_db): eval_id = (await _create_eval(client))["id"] listed = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"][0] detail = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots/{listed['id']}")).json() assert set(detail.keys()) == self.SNAPSHOT_KEYS assert detail == listed async def test_snapshot_of_other_eval_returns_404(self, client, seeded_db): eval_id = (await _create_eval(client))["id"] other_id = (await _create_eval(client, name="另一个评估"))["id"] foreign = (await client.get(f"/api/intelligent-evals/{other_id}/config-snapshots")).json()["snapshots"][0] resp = await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots/{foreign['id']}") assert resp.status_code == 404 class TestExpiredSessionMarkdownAnnotation: """ADR-0011:expired 会话在 Markdown 报告尾部标注不完整证据。""" async def test_markdown_annotates_expired_sessions(self, client, seeded_db): eval_id = await _create_executing_eval(client) session = await _create_session(client, eval_id) row = seeded_db.exec(select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.id == session["id"])).one() row.status = "expired" row.turn_count = 3 row.closed_at = utc_now() seeded_db.add(row) seeded_db.commit() await _complete_eval(client, eval_id) resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown") assert resp.status_code == 200 text = resp.text assert "## 不完整证据会话" in text assert f"- 会话 `{session['id']}`(3 轮):60 分钟无新轮次过期,证据不完整" in text async def test_markdown_without_expired_sessions_has_no_annotation(self, client, seeded_db): eval_id = await _create_executing_eval(client) await _complete_eval(client, eval_id) resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown") assert resp.status_code == 200 assert "不完整证据会话" not in resp.text