refactor(intelligent-eval): 可见性接缝收敛(Phase 1)
将「已删即 404」语义收进 IntelligentEvalRepository 单一接缝,消除三处独立裁决; 任务监控开始隐藏已删评估的任务(本 Phase 唯一刻意行为变化)。 - repository.py 新增 visible() 谓词与 require_live_eval() 服务接缝; 六处裸谓词统一走它,get()/get_including_deleted() 语义不变。 - decision_logs.py 删除本地 _require_eval,三处调用迁至 repository 接缝; count_decisions 由 len(.all()) 改为 func.count。 - task_queue.py list_tasks 与 stats 过滤已删评估的任务(行为变化)。 - web/routers/intelligent_evals.py: _require_eval_exists → _require_live_eval, 把 LookupError 翻译为 404;expired 会话 Markdown 标注下沉至 read_model.report_markdown_by_eval;配置快照 11 字段序列化收至 config_snapshot.snapshot_to_dict 单一出口。 - AGENTS.md 登记可见性纪律(已知陷阱 #6)。 - 补 characterization 测试锁定四处契约;更新 task_queue 测试以使用 真实 eval_id(可见性过滤后字面 eval_id 不再可见)。
This commit is contained in:
parent
913dc9ae86
commit
71543f042a
@ -197,3 +197,5 @@ tests/
|
|||||||
4. **SQLite 连接池**:必须用 `StaticPool` + `check_same_thread=False`。使用默认连接池会导致多线程下连接耗尽(async + FastAPI 线程池混合场景)。
|
4. **SQLite 连接池**:必须用 `StaticPool` + `check_same_thread=False`。使用默认连接池会导致多线程下连接耗尽(async + FastAPI 线程池混合场景)。
|
||||||
|
|
||||||
5. **`.env` 在 t480 上需手动维护**:部署脚本不会自动同步 `.env` 文件,新增环境变量需 SSH 到主机手动补全。容易导致 OpenClaw 代理等功能静默失效。
|
5. **`.env` 在 t480 上需手动维护**:部署脚本不会自动同步 `.env` 文件,新增环境变量需 SSH 到主机手动补全。容易导致 OpenClaw 代理等功能静默失效。
|
||||||
|
|
||||||
|
6. **智能评估可见性接缝**:逻辑删除(`deleted` 状态)语义只住在 `intelligent_eval/repository.py`——查询走 `IntelligentEvalRepository.visible()` 谓词,存在性校验走 `require_live_eval`(路由层翻译为 404)。禁止在其他模块直查 `IntelligentEvalDB` 判断删除态,否则删除语义会在多处漂移(Phase 1 收敛前的教训:快照、决策日志、任务监控各自实现了一遍"已删即 404")。
|
||||||
|
|||||||
@ -70,6 +70,23 @@ def get_snapshot(snapshot_id: str, session: Session) -> Optional[IntelligentEval
|
|||||||
return session.get(IntelligentEvalConfigSnapshotDB, snapshot_id)
|
return session.get(IntelligentEvalConfigSnapshotDB, snapshot_id)
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot_to_dict(s: IntelligentEvalConfigSnapshotDB) -> dict[str, Any]:
|
||||||
|
"""快照的稳定序列化形状(列表与详情共用的单一出口)。"""
|
||||||
|
return {
|
||||||
|
"id": s.id,
|
||||||
|
"eval_id": s.eval_id,
|
||||||
|
"snapshot_type": s.snapshot_type,
|
||||||
|
"goal": s.goal,
|
||||||
|
"seeds": s.get_seeds(),
|
||||||
|
"intent": s.intent,
|
||||||
|
"role_description": s.role_description,
|
||||||
|
"time_window_hours": s.time_window_hours,
|
||||||
|
"plan": s.get_plan(),
|
||||||
|
"created_at": s.created_at.isoformat() if s.created_at else None,
|
||||||
|
"created_by": s.created_by,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def compare_snapshots(
|
def compare_snapshots(
|
||||||
snapshot1: IntelligentEvalConfigSnapshotDB,
|
snapshot1: IntelligentEvalConfigSnapshotDB,
|
||||||
snapshot2: IntelligentEvalConfigSnapshotDB,
|
snapshot2: IntelligentEvalConfigSnapshotDB,
|
||||||
|
|||||||
@ -3,11 +3,14 @@
|
|||||||
Pulled out of ``web/routers/intelligent_evals.py`` so the router only handles
|
Pulled out of ``web/routers/intelligent_evals.py`` so the router only handles
|
||||||
HTTP validation and error translation. The ORM writes and reads now live here.
|
HTTP validation and error translation. The ORM writes and reads now live here.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import func
|
||||||
from sqlmodel import Session, select
|
from sqlmodel import Session, select
|
||||||
|
|
||||||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||||||
|
from agenteval.intelligent_eval.repository import IntelligentEvalRepository
|
||||||
from agenteval.storage.db import (
|
from agenteval.storage.db import (
|
||||||
IntelligentEvalDB,
|
IntelligentEvalDB,
|
||||||
IntelligentEvalDecisionLogDB,
|
IntelligentEvalDecisionLogDB,
|
||||||
@ -27,12 +30,6 @@ def _log_to_dict(log: IntelligentEvalDecisionLogDB) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _require_eval(eval_id: str, session: Session) -> None:
|
|
||||||
eval_db = session.get(IntelligentEvalDB, eval_id)
|
|
||||||
if eval_db is None or eval_db.status == IntelligentEvalStatus.DELETED.value:
|
|
||||||
raise LookupError(f"intelligent eval {eval_id} not found")
|
|
||||||
|
|
||||||
|
|
||||||
def _append_row(
|
def _append_row(
|
||||||
eval_id: str,
|
eval_id: str,
|
||||||
decision_type: str,
|
decision_type: str,
|
||||||
@ -71,7 +68,7 @@ def create_decision_log(
|
|||||||
|
|
||||||
Raises ``LookupError`` if eval not found.
|
Raises ``LookupError`` if eval not found.
|
||||||
"""
|
"""
|
||||||
_require_eval(eval_id, session)
|
IntelligentEvalRepository(session).require_live_eval(eval_id)
|
||||||
|
|
||||||
# Dedupe: same (eval, decision_type, context) → return existing.
|
# Dedupe: same (eval, decision_type, context) → return existing.
|
||||||
# Limit to 100 records to avoid loading too many into memory; in practice,
|
# Limit to 100 records to avoid loading too many into memory; in practice,
|
||||||
@ -105,25 +102,25 @@ def append_decision_log(
|
|||||||
|
|
||||||
Raises ``LookupError`` if eval not found.
|
Raises ``LookupError`` if eval not found.
|
||||||
"""
|
"""
|
||||||
_require_eval(eval_id, session)
|
IntelligentEvalRepository(session).require_live_eval(eval_id)
|
||||||
return _append_row(eval_id, decision_type, reason, cron_id, context, session)
|
return _append_row(eval_id, decision_type, reason, cron_id, context, session)
|
||||||
|
|
||||||
|
|
||||||
def count_decisions(eval_id: str, decision_type: str, session: Session) -> int:
|
def count_decisions(eval_id: str, decision_type: str, session: Session) -> int:
|
||||||
"""该评估某类型决策日志的条数(平台闸门计数与 attempt 落账的计数原语)。"""
|
"""该评估某类型决策日志的条数(平台闸门计数与 attempt 落账的计数原语)。"""
|
||||||
return len(
|
return session.exec(
|
||||||
session.exec(
|
select(func.count())
|
||||||
select(IntelligentEvalDecisionLogDB).where(
|
.select_from(IntelligentEvalDecisionLogDB)
|
||||||
IntelligentEvalDecisionLogDB.eval_id == eval_id,
|
.where(
|
||||||
IntelligentEvalDecisionLogDB.decision_type == decision_type,
|
IntelligentEvalDecisionLogDB.eval_id == eval_id,
|
||||||
)
|
IntelligentEvalDecisionLogDB.decision_type == decision_type,
|
||||||
).all()
|
)
|
||||||
)
|
).one()
|
||||||
|
|
||||||
|
|
||||||
def list_decision_logs(eval_id: str, session: Session) -> list[dict]:
|
def list_decision_logs(eval_id: str, session: Session) -> list[dict]:
|
||||||
"""List decision logs for an eval. Raises ``LookupError`` if eval not found."""
|
"""List decision logs for an eval. Raises ``LookupError`` if eval not found."""
|
||||||
_require_eval(eval_id, session)
|
IntelligentEvalRepository(session).require_live_eval(eval_id)
|
||||||
|
|
||||||
logs = session.exec(
|
logs = session.exec(
|
||||||
select(IntelligentEvalDecisionLogDB)
|
select(IntelligentEvalDecisionLogDB)
|
||||||
@ -252,9 +249,7 @@ def supplement_decision_logs(session: Session) -> int:
|
|||||||
for ev in evals:
|
for ev in evals:
|
||||||
plan = ev.get_plan() if ev.plan else {}
|
plan = ev.get_plan() if ev.plan else {}
|
||||||
estimated = plan.get("estimated_sessions", 0)
|
estimated = plan.get("estimated_sessions", 0)
|
||||||
sessions = session.exec(
|
sessions = session.exec(select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.eval_id == ev.id)).all()
|
||||||
select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.eval_id == ev.id)
|
|
||||||
).all()
|
|
||||||
completed = sum(1 for s in sessions if s.status == "completed")
|
completed = sum(1 for s in sessions if s.status == "completed")
|
||||||
types = {
|
types = {
|
||||||
x.decision_type
|
x.decision_type
|
||||||
|
|||||||
@ -22,6 +22,7 @@ from agenteval.intelligent_eval.models import (
|
|||||||
IntelligentEvalSessionStatus,
|
IntelligentEvalSessionStatus,
|
||||||
IntelligentEvalStatus,
|
IntelligentEvalStatus,
|
||||||
)
|
)
|
||||||
|
from agenteval.intelligent_eval.report import render_report_markdown
|
||||||
from agenteval.intelligent_eval.repository import (
|
from agenteval.intelligent_eval.repository import (
|
||||||
IntelligentEvalMessageRepository,
|
IntelligentEvalMessageRepository,
|
||||||
IntelligentEvalRepository,
|
IntelligentEvalRepository,
|
||||||
@ -345,3 +346,19 @@ class IntelligentEvalReadModel:
|
|||||||
if eval_obj is None or eval_obj.report is None:
|
if eval_obj is None or eval_obj.report is None:
|
||||||
return None
|
return None
|
||||||
return eval_obj.name, eval_obj.report
|
return eval_obj.name, eval_obj.report
|
||||||
|
|
||||||
|
def report_markdown_by_eval(self, eval_id: str) -> str | None:
|
||||||
|
"""报告的 Markdown 投影(含 expired 会话的不完整证据标注,ADR-0011)。"""
|
||||||
|
|
||||||
|
report = self.report_by_eval(eval_id)
|
||||||
|
if report is None:
|
||||||
|
return None
|
||||||
|
name, payload = report
|
||||||
|
markdown = render_report_markdown(payload, name=name, eval_id=eval_id)
|
||||||
|
expired = [s for s in self._sessions.list_by_eval(eval_id) if s.status is IntelligentEvalSessionStatus.EXPIRED]
|
||||||
|
if expired:
|
||||||
|
lines = ["", "## 不完整证据会话", ""]
|
||||||
|
for s in expired:
|
||||||
|
lines.append(f"- 会话 `{s.id}`({s.turn_count} 轮):60 分钟无新轮次过期,证据不完整")
|
||||||
|
markdown = markdown.rstrip() + "\n" + "\n".join(lines) + "\n"
|
||||||
|
return markdown
|
||||||
|
|||||||
@ -97,26 +97,25 @@ class IntelligentEvalRepository:
|
|||||||
completed_at=db.completed_at,
|
completed_at=db.completed_at,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def visible():
|
||||||
|
"""可见性谓词:逻辑删除的评估默认不可见(查询位点共用接缝)。"""
|
||||||
|
return IntelligentEvalDB.status != IntelligentEvalStatus.DELETED.value
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_visible(db: IntelligentEvalDB) -> bool:
|
||||||
|
return db.status != IntelligentEvalStatus.DELETED.value
|
||||||
|
|
||||||
def list_all(self) -> list[IntelligentEval]:
|
def list_all(self) -> list[IntelligentEval]:
|
||||||
statement = (
|
statement = select(IntelligentEvalDB).where(self.visible()).order_by(IntelligentEvalDB.created_at.desc())
|
||||||
select(IntelligentEvalDB)
|
|
||||||
.where(IntelligentEvalDB.status != IntelligentEvalStatus.DELETED.value)
|
|
||||||
.order_by(IntelligentEvalDB.created_at.desc())
|
|
||||||
)
|
|
||||||
return [self._from_db(r) for r in self.session.exec(statement).all()]
|
return [self._from_db(r) for r in self.session.exec(statement).all()]
|
||||||
|
|
||||||
def list_page(
|
def list_page(self, offset: int, limit: int, status: Optional[str] = None) -> list[IntelligentEval]:
|
||||||
self, offset: int, limit: int, status: Optional[str] = None
|
|
||||||
) -> list[IntelligentEval]:
|
|
||||||
"""Return one page of evals (created_at desc), optionally filtered by status.
|
"""Return one page of evals (created_at desc), optionally filtered by status.
|
||||||
|
|
||||||
Deleted evals are always excluded — the list API has no recycle bin.
|
Deleted evals are always excluded — the list API has no recycle bin.
|
||||||
"""
|
"""
|
||||||
statement = (
|
statement = select(IntelligentEvalDB).where(self.visible()).order_by(IntelligentEvalDB.created_at.desc())
|
||||||
select(IntelligentEvalDB)
|
|
||||||
.where(IntelligentEvalDB.status != IntelligentEvalStatus.DELETED.value)
|
|
||||||
.order_by(IntelligentEvalDB.created_at.desc())
|
|
||||||
)
|
|
||||||
if status:
|
if status:
|
||||||
statement = statement.where(IntelligentEvalDB.status == status)
|
statement = statement.where(IntelligentEvalDB.status == status)
|
||||||
statement = statement.offset(offset).limit(limit)
|
statement = statement.offset(offset).limit(limit)
|
||||||
@ -124,17 +123,13 @@ class IntelligentEvalRepository:
|
|||||||
|
|
||||||
def count(self) -> int:
|
def count(self) -> int:
|
||||||
"""Total number of evaluations (for pagination metadata)."""
|
"""Total number of evaluations (for pagination metadata)."""
|
||||||
return self.session.exec(
|
return self.session.exec(select(func.count()).select_from(IntelligentEvalDB).where(self.visible())).one()
|
||||||
select(func.count())
|
|
||||||
.select_from(IntelligentEvalDB)
|
|
||||||
.where(IntelligentEvalDB.status != IntelligentEvalStatus.DELETED.value)
|
|
||||||
).one()
|
|
||||||
|
|
||||||
def count_by_status(self) -> dict[str, int]:
|
def count_by_status(self) -> dict[str, int]:
|
||||||
"""Count evaluations per status (for the list page stat bar)."""
|
"""Count evaluations per status (for the list page stat bar)."""
|
||||||
rows = self.session.exec(
|
rows = self.session.exec(
|
||||||
select(IntelligentEvalDB.status, func.count(IntelligentEvalDB.id))
|
select(IntelligentEvalDB.status, func.count(IntelligentEvalDB.id))
|
||||||
.where(IntelligentEvalDB.status != IntelligentEvalStatus.DELETED.value)
|
.where(self.visible())
|
||||||
.group_by(IntelligentEvalDB.status)
|
.group_by(IntelligentEvalDB.status)
|
||||||
).all()
|
).all()
|
||||||
stats = {status.value: 0 for status in IntelligentEvalStatus if status != IntelligentEvalStatus.DELETED}
|
stats = {status.value: 0 for status in IntelligentEvalStatus if status != IntelligentEvalStatus.DELETED}
|
||||||
@ -144,10 +139,17 @@ class IntelligentEvalRepository:
|
|||||||
|
|
||||||
def get(self, eval_id: str) -> Optional[IntelligentEval]:
|
def get(self, eval_id: str) -> Optional[IntelligentEval]:
|
||||||
db = self.session.get(IntelligentEvalDB, eval_id)
|
db = self.session.get(IntelligentEvalDB, eval_id)
|
||||||
if db is None or db.status == IntelligentEvalStatus.DELETED.value:
|
if db is None or not self._is_visible(db):
|
||||||
return None
|
return None
|
||||||
return self._from_db(db)
|
return self._from_db(db)
|
||||||
|
|
||||||
|
def require_live_eval(self, eval_id: str) -> IntelligentEval:
|
||||||
|
"""可见性单一接缝:存在且未删 → 返回实体,否则 ``LookupError``。"""
|
||||||
|
ev = self.get(eval_id)
|
||||||
|
if ev is None:
|
||||||
|
raise LookupError(f"intelligent eval {eval_id} not found")
|
||||||
|
return ev
|
||||||
|
|
||||||
def get_including_deleted(self, eval_id: str) -> Optional[IntelligentEval]:
|
def get_including_deleted(self, eval_id: str) -> Optional[IntelligentEval]:
|
||||||
"""包含已删除评估的原样读取(供 delete 幂等检查使用)。"""
|
"""包含已删除评估的原样读取(供 delete 幂等检查使用)。"""
|
||||||
db = self.session.get(IntelligentEvalDB, eval_id)
|
db = self.session.get(IntelligentEvalDB, eval_id)
|
||||||
@ -168,7 +170,7 @@ class IntelligentEvalRepository:
|
|||||||
)
|
)
|
||||||
.where(
|
.where(
|
||||||
IntelligentEvalDB.id == eval_id,
|
IntelligentEvalDB.id == eval_id,
|
||||||
IntelligentEvalDB.status != IntelligentEvalStatus.DELETED.value,
|
self.visible(),
|
||||||
)
|
)
|
||||||
.order_by(IntelligentEvalSessionDB.created_at.asc())
|
.order_by(IntelligentEvalSessionDB.created_at.asc())
|
||||||
)
|
)
|
||||||
@ -285,6 +287,7 @@ class IntelligentEvalRepository:
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class IntelligentEvalSessionRepository:
|
class IntelligentEvalSessionRepository:
|
||||||
"""CRUD for intelligent eval sessions."""
|
"""CRUD for intelligent eval sessions."""
|
||||||
|
|
||||||
@ -443,6 +446,7 @@ class IntelligentEvalSessionRepository:
|
|||||||
return CompareAndSetStatus.NOT_FOUND, None
|
return CompareAndSetStatus.NOT_FOUND, None
|
||||||
return CompareAndSetStatus.APPLIED, self._from_db(db)
|
return CompareAndSetStatus.APPLIED, self._from_db(db)
|
||||||
|
|
||||||
|
|
||||||
class IntelligentEvalMessageRepository:
|
class IntelligentEvalMessageRepository:
|
||||||
"""CRUD for intelligent eval session messages."""
|
"""CRUD for intelligent eval session messages."""
|
||||||
|
|
||||||
|
|||||||
@ -13,6 +13,7 @@ from typing import Optional
|
|||||||
from sqlmodel import Session, func, select, update
|
from sqlmodel import Session, func, select, update
|
||||||
|
|
||||||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||||||
|
from agenteval.intelligent_eval.repository import IntelligentEvalRepository
|
||||||
from agenteval.storage.db import (
|
from agenteval.storage.db import (
|
||||||
IntelligentEvalDB,
|
IntelligentEvalDB,
|
||||||
IntelligentEvalTaskQueueDB,
|
IntelligentEvalTaskQueueDB,
|
||||||
@ -259,9 +260,7 @@ def settle_tasks_for_finished_evals(session: Session) -> int:
|
|||||||
from agenteval.storage.db import IntelligentEvalTaskQueueDB
|
from agenteval.storage.db import IntelligentEvalTaskQueueDB
|
||||||
|
|
||||||
tasks = session.exec(
|
tasks = session.exec(
|
||||||
select(IntelligentEvalTaskQueueDB).where(
|
select(IntelligentEvalTaskQueueDB).where(IntelligentEvalTaskQueueDB.status.in_(["pending", "assigned"]))
|
||||||
IntelligentEvalTaskQueueDB.status.in_(["pending", "assigned"])
|
|
||||||
)
|
|
||||||
).all()
|
).all()
|
||||||
|
|
||||||
settled = 0
|
settled = 0
|
||||||
@ -305,7 +304,14 @@ def list_tasks(
|
|||||||
Returns:
|
Returns:
|
||||||
{"tasks": [...], "stats": {pending, assigned, completed, failed, unresolved}}
|
{"tasks": [...], "stats": {pending, assigned, completed, failed, unresolved}}
|
||||||
"""
|
"""
|
||||||
stmt = select(IntelligentEvalTaskQueueDB).order_by(IntelligentEvalTaskQueueDB.created_at.desc())
|
# 可见性接缝:已删评估的任务不出现在监控列表与统计中
|
||||||
|
live_eval_ids = select(IntelligentEvalDB.id).where(IntelligentEvalRepository.visible())
|
||||||
|
|
||||||
|
stmt = (
|
||||||
|
select(IntelligentEvalTaskQueueDB)
|
||||||
|
.where(IntelligentEvalTaskQueueDB.eval_id.in_(live_eval_ids))
|
||||||
|
.order_by(IntelligentEvalTaskQueueDB.created_at.desc())
|
||||||
|
)
|
||||||
if status:
|
if status:
|
||||||
stmt = stmt.where(IntelligentEvalTaskQueueDB.status == status)
|
stmt = stmt.where(IntelligentEvalTaskQueueDB.status == status)
|
||||||
if eval_id:
|
if eval_id:
|
||||||
@ -318,7 +324,9 @@ def list_tasks(
|
|||||||
select(
|
select(
|
||||||
IntelligentEvalTaskQueueDB.status,
|
IntelligentEvalTaskQueueDB.status,
|
||||||
func.count(IntelligentEvalTaskQueueDB.id),
|
func.count(IntelligentEvalTaskQueueDB.id),
|
||||||
).group_by(IntelligentEvalTaskQueueDB.status)
|
)
|
||||||
|
.where(IntelligentEvalTaskQueueDB.eval_id.in_(live_eval_ids))
|
||||||
|
.group_by(IntelligentEvalTaskQueueDB.status)
|
||||||
).all()
|
).all()
|
||||||
for status_val, cnt in rows:
|
for status_val, cnt in rows:
|
||||||
if status_val in stats:
|
if status_val in stats:
|
||||||
|
|||||||
@ -17,10 +17,9 @@ from agenteval.intelligent_eval.lifecycle import (
|
|||||||
IntelligentEvalNotFoundError,
|
IntelligentEvalNotFoundError,
|
||||||
IntelligentEvalTransitionError,
|
IntelligentEvalTransitionError,
|
||||||
)
|
)
|
||||||
from agenteval.intelligent_eval.models import IntelligentEvalSessionStatus
|
|
||||||
from agenteval.intelligent_eval.read_model import IntelligentEvalReadModel
|
from agenteval.intelligent_eval.read_model import IntelligentEvalReadModel
|
||||||
from agenteval.intelligent_eval.report import ReportModel, render_report_markdown
|
from agenteval.intelligent_eval.report import ReportModel
|
||||||
from agenteval.intelligent_eval.repository import IntelligentEvalSessionRepository
|
from agenteval.intelligent_eval.repository import IntelligentEvalRepository
|
||||||
from agenteval.web.deps import get_db
|
from agenteval.web.deps import get_db
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
@ -70,14 +69,12 @@ def _translate(exc: Exception) -> HTTPException:
|
|||||||
return HTTPException(status_code=409, detail=exc.reason)
|
return HTTPException(status_code=409, detail=exc.reason)
|
||||||
|
|
||||||
|
|
||||||
def _require_eval_exists(session: Session, eval_id: str) -> None:
|
def _require_live_eval(session: Session, eval_id: str) -> None:
|
||||||
"""404 when the eval is missing or logically deleted."""
|
"""把可见性接缝翻译成 HTTP:缺失或已删 → 404。"""
|
||||||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
try:
|
||||||
from agenteval.storage.db import IntelligentEvalDB
|
IntelligentEvalRepository(session).require_live_eval(eval_id)
|
||||||
|
except LookupError as exc:
|
||||||
eval_db = session.get(IntelligentEvalDB, eval_id)
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||||
if eval_db is None or eval_db.status == IntelligentEvalStatus.DELETED.value:
|
|
||||||
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
|
|
||||||
|
|
||||||
|
|
||||||
def _eval_response(ev, session: Session) -> dict:
|
def _eval_response(ev, session: Session) -> dict:
|
||||||
@ -234,22 +231,9 @@ async def get_report(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
|||||||
|
|
||||||
@router.get("/{eval_id}/report/markdown", response_class=PlainTextResponse)
|
@router.get("/{eval_id}/report/markdown", response_class=PlainTextResponse)
|
||||||
async def get_report_markdown(eval_id: str, session: Session = Depends(get_db)) -> PlainTextResponse:
|
async def get_report_markdown(eval_id: str, session: Session = Depends(get_db)) -> PlainTextResponse:
|
||||||
report = IntelligentEvalReadModel(session).report_by_eval(eval_id)
|
markdown = IntelligentEvalReadModel(session).report_markdown_by_eval(eval_id)
|
||||||
if report is None:
|
if markdown is None:
|
||||||
raise HTTPException(status_code=404, detail="report not submitted yet")
|
raise HTTPException(status_code=404, detail="report not submitted yet")
|
||||||
name, payload = report
|
|
||||||
markdown = render_report_markdown(payload, name=name, eval_id=eval_id)
|
|
||||||
# ADR-0011:expired 会话按不完整证据标注,提醒读者结论证据不全
|
|
||||||
expired_sessions = [
|
|
||||||
s
|
|
||||||
for s in IntelligentEvalSessionRepository(session).list_by_eval(eval_id)
|
|
||||||
if s.status == IntelligentEvalSessionStatus.EXPIRED
|
|
||||||
]
|
|
||||||
if expired_sessions:
|
|
||||||
lines = ["", "## 不完整证据会话", ""]
|
|
||||||
for s in expired_sessions:
|
|
||||||
lines.append(f"- 会话 `{s.id}`({s.turn_count} 轮):60 分钟无新轮次过期,证据不完整")
|
|
||||||
markdown = markdown.rstrip() + "\n" + "\n".join(lines) + "\n"
|
|
||||||
return PlainTextResponse(markdown, media_type="text/markdown; charset=utf-8")
|
return PlainTextResponse(markdown, media_type="text/markdown; charset=utf-8")
|
||||||
|
|
||||||
|
|
||||||
@ -393,27 +377,10 @@ async def list_config_snapshots(eval_id: str, session: Session = Depends(get_db)
|
|||||||
"""List all config snapshots for an evaluation."""
|
"""List all config snapshots for an evaluation."""
|
||||||
from agenteval.intelligent_eval import config_snapshot
|
from agenteval.intelligent_eval import config_snapshot
|
||||||
|
|
||||||
_require_eval_exists(session, eval_id)
|
_require_live_eval(session, eval_id)
|
||||||
|
|
||||||
snapshots = config_snapshot.list_snapshots(eval_id, session)
|
snapshots = config_snapshot.list_snapshots(eval_id, session)
|
||||||
return {
|
return {"snapshots": [config_snapshot.snapshot_to_dict(s) for s in snapshots]}
|
||||||
"snapshots": [
|
|
||||||
{
|
|
||||||
"id": s.id,
|
|
||||||
"eval_id": s.eval_id,
|
|
||||||
"snapshot_type": s.snapshot_type,
|
|
||||||
"goal": s.goal,
|
|
||||||
"seeds": s.get_seeds(),
|
|
||||||
"intent": s.intent,
|
|
||||||
"role_description": s.role_description,
|
|
||||||
"time_window_hours": s.time_window_hours,
|
|
||||||
"plan": s.get_plan(),
|
|
||||||
"created_at": s.created_at.isoformat() if s.created_at else None,
|
|
||||||
"created_by": s.created_by,
|
|
||||||
}
|
|
||||||
for s in snapshots
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{eval_id}/config-snapshots/{snapshot_id}")
|
@router.get("/{eval_id}/config-snapshots/{snapshot_id}")
|
||||||
@ -421,25 +388,13 @@ async def get_config_snapshot(eval_id: str, snapshot_id: str, session: Session =
|
|||||||
"""Get a single config snapshot."""
|
"""Get a single config snapshot."""
|
||||||
from agenteval.intelligent_eval import config_snapshot
|
from agenteval.intelligent_eval import config_snapshot
|
||||||
|
|
||||||
_require_eval_exists(session, eval_id)
|
_require_live_eval(session, eval_id)
|
||||||
|
|
||||||
snapshot = config_snapshot.get_snapshot(snapshot_id, session)
|
snapshot = config_snapshot.get_snapshot(snapshot_id, session)
|
||||||
if snapshot is None or snapshot.eval_id != eval_id:
|
if snapshot is None or snapshot.eval_id != eval_id:
|
||||||
raise HTTPException(status_code=404, detail=f"snapshot {snapshot_id} not found")
|
raise HTTPException(status_code=404, detail=f"snapshot {snapshot_id} not found")
|
||||||
|
|
||||||
return {
|
return config_snapshot.snapshot_to_dict(snapshot)
|
||||||
"id": snapshot.id,
|
|
||||||
"eval_id": snapshot.eval_id,
|
|
||||||
"snapshot_type": snapshot.snapshot_type,
|
|
||||||
"goal": snapshot.goal,
|
|
||||||
"seeds": snapshot.get_seeds(),
|
|
||||||
"intent": snapshot.intent,
|
|
||||||
"role_description": snapshot.role_description,
|
|
||||||
"time_window_hours": snapshot.time_window_hours,
|
|
||||||
"plan": snapshot.get_plan(),
|
|
||||||
"created_at": snapshot.created_at.isoformat() if snapshot.created_at else None,
|
|
||||||
"created_by": snapshot.created_by,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class CompareSnapshotsRequest(BaseModel):
|
class CompareSnapshotsRequest(BaseModel):
|
||||||
@ -456,7 +411,7 @@ async def compare_config_snapshots(
|
|||||||
"""Compare two config snapshots and return differences."""
|
"""Compare two config snapshots and return differences."""
|
||||||
from agenteval.intelligent_eval import config_snapshot
|
from agenteval.intelligent_eval import config_snapshot
|
||||||
|
|
||||||
_require_eval_exists(session, eval_id)
|
_require_live_eval(session, eval_id)
|
||||||
|
|
||||||
# Get both snapshots
|
# Get both snapshots
|
||||||
snapshot1 = config_snapshot.get_snapshot(request.snapshot_id_1, session)
|
snapshot1 = config_snapshot.get_snapshot(request.snapshot_id_1, session)
|
||||||
|
|||||||
@ -37,6 +37,7 @@ def db_session(tmp_db_path: Path) -> Session:
|
|||||||
EvalTargetDB,
|
EvalTargetDB,
|
||||||
ExplorationMessageDB,
|
ExplorationMessageDB,
|
||||||
ExplorationSessionDB,
|
ExplorationSessionDB,
|
||||||
|
IntelligentEvalConfigSnapshotDB,
|
||||||
IntelligentEvalDB,
|
IntelligentEvalDB,
|
||||||
IntelligentEvalDecisionLogDB,
|
IntelligentEvalDecisionLogDB,
|
||||||
IntelligentEvalMessageDB,
|
IntelligentEvalMessageDB,
|
||||||
|
|||||||
8
tests/integration/conftest.py
Normal file
8
tests/integration/conftest.py
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
"""Integration-level shared fixtures.
|
||||||
|
|
||||||
|
Re-exports the in-process app fixtures from ``test_intelligent_evals_api``
|
||||||
|
so newer integration test modules can consume them without importing
|
||||||
|
fixture objects into test-module namespaces (which ruff flags as F811).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .test_intelligent_evals_api import client, seeded_db # noqa: F401
|
||||||
@ -257,8 +257,9 @@ def test_list_tasks(client: TestClient, db_session: Session):
|
|||||||
|
|
||||||
def test_list_tasks_status_filter(client: TestClient, db_session: Session):
|
def test_list_tasks_status_filter(client: TestClient, db_session: Session):
|
||||||
"""Status filter narrows the task list."""
|
"""Status filter narrows the task list."""
|
||||||
db_session.add(IntelligentEvalTaskQueueDB(eval_id="eval1", status="pending", priority=1, reason="slot_due"))
|
eval_db = _make_eval(db_session, "status-filter-eval", days_ago=0)
|
||||||
db_session.add(IntelligentEvalTaskQueueDB(eval_id="eval1", status="failed", priority=1, reason="slot_due"))
|
db_session.add(IntelligentEvalTaskQueueDB(eval_id=eval_db.id, status="pending", priority=1, reason="slot_due"))
|
||||||
|
db_session.add(IntelligentEvalTaskQueueDB(eval_id=eval_db.id, status="failed", priority=1, reason="slot_due"))
|
||||||
db_session.commit()
|
db_session.commit()
|
||||||
|
|
||||||
response = client.get("/api/intelligent-evals/tasks?status=failed")
|
response = client.get("/api/intelligent-evals/tasks?status=failed")
|
||||||
|
|||||||
@ -0,0 +1,204 @@
|
|||||||
|
"""Characterization tests: intelligent-eval visibility semantics (Phase 1).
|
||||||
|
|
||||||
|
锁定逻辑删除可见性收敛(require_live_eval 接缝)后的行为契约。覆盖四处:
|
||||||
|
|
||||||
|
1. 「已删即 404」的统一裁决:config-snapshots 详情/对比端点与
|
||||||
|
decision_logs 服务层都经 require_live_eval 接缝。
|
||||||
|
2. 任务队列监控**隐藏**已删评估的任务(Phase 1.6 唯一的刻意行为变化,
|
||||||
|
锁定变更后的新契约)。
|
||||||
|
3. config-snapshots 端点的响应形状(11 字段序列化,snapshot_to_dict 单一出口)。
|
||||||
|
4. expired 会话在 Markdown 报告中的「不完整证据」标注文本(ADR-0011)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||||||
|
from agenteval.storage.db import (
|
||||||
|
IntelligentEvalSessionDB,
|
||||||
|
IntelligentEvalTaskQueueDB,
|
||||||
|
utc_now,
|
||||||
|
)
|
||||||
|
from sqlmodel import select
|
||||||
|
|
||||||
|
from .test_intelligent_evals_api import (
|
||||||
|
_create_eval,
|
||||||
|
_create_executing_eval,
|
||||||
|
_create_session,
|
||||||
|
_report_payload,
|
||||||
|
_submit_plan,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _complete_eval(client, eval_id: str) -> None:
|
||||||
|
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
||||||
|
assert resp.status_code == 200, resp.text
|
||||||
|
|
||||||
|
|
||||||
|
class TestSnapshotEndpointDeletedSemantics:
|
||||||
|
"""路由 _require_eval_exists 路径:config-snapshots 详情与对比。"""
|
||||||
|
|
||||||
|
async def test_deleted_eval_snapshot_detail_returns_404(self, client, seeded_db):
|
||||||
|
eval_id = await _create_executing_eval(client)
|
||||||
|
await _complete_eval(client, eval_id)
|
||||||
|
snapshots = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"]
|
||||||
|
assert snapshots
|
||||||
|
await client.delete(f"/api/intelligent-evals/{eval_id}")
|
||||||
|
|
||||||
|
resp = await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots/{snapshots[0]['id']}")
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|
||||||
|
async def test_deleted_eval_snapshot_compare_returns_404(self, client, seeded_db):
|
||||||
|
eval_id = (await _create_eval(client))["id"]
|
||||||
|
await _submit_plan(client, eval_id) # second snapshot (plan_submitted)
|
||||||
|
await client.post(f"/api/intelligent-evals/{eval_id}/approve")
|
||||||
|
await _complete_eval(client, eval_id)
|
||||||
|
snapshots = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"]
|
||||||
|
assert len(snapshots) >= 2
|
||||||
|
await client.delete(f"/api/intelligent-evals/{eval_id}")
|
||||||
|
|
||||||
|
resp = await client.post(
|
||||||
|
f"/api/intelligent-evals/{eval_id}/config-snapshots/compare",
|
||||||
|
json={"snapshot_id_1": snapshots[0]["id"], "snapshot_id_2": snapshots[1]["id"]},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
class TestDecisionLogServiceDeletedSemantics:
|
||||||
|
"""decision_logs 经 require_live_eval 接缝:已删评估一律 LookupError。"""
|
||||||
|
|
||||||
|
async def test_deleted_eval_service_calls_raise_lookup_error(self, client, seeded_db):
|
||||||
|
from agenteval.intelligent_eval import decision_logs
|
||||||
|
|
||||||
|
eval_id = await _create_executing_eval(client)
|
||||||
|
await _complete_eval(client, eval_id)
|
||||||
|
await client.delete(f"/api/intelligent-evals/{eval_id}")
|
||||||
|
|
||||||
|
with pytest.raises(LookupError):
|
||||||
|
decision_logs.list_decision_logs(eval_id, seeded_db)
|
||||||
|
with pytest.raises(LookupError):
|
||||||
|
decision_logs.create_decision_log(eval_id, "wait", "late report", "w-1", {}, seeded_db)
|
||||||
|
with pytest.raises(LookupError):
|
||||||
|
decision_logs.append_decision_log(eval_id, "wait", "late bookkeeping", "platform", {}, seeded_db)
|
||||||
|
|
||||||
|
async def test_repo_get_hides_deleted_but_including_deleted_returns_it(self, client, seeded_db):
|
||||||
|
from agenteval.intelligent_eval.repository import IntelligentEvalRepository
|
||||||
|
|
||||||
|
eval_id = await _create_executing_eval(client)
|
||||||
|
await _complete_eval(client, eval_id)
|
||||||
|
await client.delete(f"/api/intelligent-evals/{eval_id}")
|
||||||
|
|
||||||
|
repo = IntelligentEvalRepository(seeded_db)
|
||||||
|
assert repo.get(eval_id) is None
|
||||||
|
hidden = repo.get_including_deleted(eval_id)
|
||||||
|
assert hidden is not None
|
||||||
|
assert hidden.status is IntelligentEvalStatus.DELETED
|
||||||
|
|
||||||
|
|
||||||
|
class TestTaskMonitorHidesDeletedEvals:
|
||||||
|
"""Phase 1.6 刻意行为变化:任务监控不再泄漏已删评估的任务。"""
|
||||||
|
|
||||||
|
async def test_list_tasks_hides_tasks_of_deleted_eval(self, client, seeded_db):
|
||||||
|
eval_id = await _create_executing_eval(client)
|
||||||
|
seeded_db.add(
|
||||||
|
IntelligentEvalTaskQueueDB(
|
||||||
|
eval_id=eval_id,
|
||||||
|
status="completed",
|
||||||
|
priority=1,
|
||||||
|
reason="slot_due",
|
||||||
|
completed_at=utc_now(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
seeded_db.commit()
|
||||||
|
|
||||||
|
await _complete_eval(client, eval_id)
|
||||||
|
await client.delete(f"/api/intelligent-evals/{eval_id}")
|
||||||
|
|
||||||
|
resp = await client.get("/api/intelligent-evals/tasks")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
data = resp.json()
|
||||||
|
|
||||||
|
assert [t for t in data["tasks"] if t["eval_id"] == eval_id] == []
|
||||||
|
assert data["stats"]["completed"] == 0
|
||||||
|
|
||||||
|
by_eval = await client.get("/api/intelligent-evals/tasks", params={"eval_id": eval_id})
|
||||||
|
assert by_eval.json()["tasks"] == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestSnapshotResponseShape:
|
||||||
|
"""锁定 config-snapshots 列表/详情的 11 字段序列化形状。"""
|
||||||
|
|
||||||
|
SNAPSHOT_KEYS = {
|
||||||
|
"id",
|
||||||
|
"eval_id",
|
||||||
|
"snapshot_type",
|
||||||
|
"goal",
|
||||||
|
"seeds",
|
||||||
|
"intent",
|
||||||
|
"role_description",
|
||||||
|
"time_window_hours",
|
||||||
|
"plan",
|
||||||
|
"created_at",
|
||||||
|
"created_by",
|
||||||
|
}
|
||||||
|
|
||||||
|
async def test_list_shape_and_snapshot_types(self, client, seeded_db):
|
||||||
|
eval_id = (await _create_eval(client))["id"]
|
||||||
|
await _submit_plan(client, eval_id)
|
||||||
|
|
||||||
|
snapshots = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"]
|
||||||
|
assert len(snapshots) == 2
|
||||||
|
assert {s["snapshot_type"] for s in snapshots} == {"created", "plan_submitted"}
|
||||||
|
for s in snapshots:
|
||||||
|
assert set(s.keys()) == self.SNAPSHOT_KEYS
|
||||||
|
assert s["eval_id"] == eval_id
|
||||||
|
created = next(s for s in snapshots if s["snapshot_type"] == "created")
|
||||||
|
assert created["created_by"] == "user"
|
||||||
|
assert created["goal"] == "评估退货流程处理能力"
|
||||||
|
assert created["time_window_hours"] == 24
|
||||||
|
assert created["plan"] is None
|
||||||
|
|
||||||
|
async def test_detail_shape_matches_list_item(self, client, seeded_db):
|
||||||
|
eval_id = (await _create_eval(client))["id"]
|
||||||
|
listed = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots")).json()["snapshots"][0]
|
||||||
|
|
||||||
|
detail = (await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots/{listed['id']}")).json()
|
||||||
|
assert set(detail.keys()) == self.SNAPSHOT_KEYS
|
||||||
|
assert detail == listed
|
||||||
|
|
||||||
|
async def test_snapshot_of_other_eval_returns_404(self, client, seeded_db):
|
||||||
|
eval_id = (await _create_eval(client))["id"]
|
||||||
|
other_id = (await _create_eval(client, name="另一个评估"))["id"]
|
||||||
|
foreign = (await client.get(f"/api/intelligent-evals/{other_id}/config-snapshots")).json()["snapshots"][0]
|
||||||
|
|
||||||
|
resp = await client.get(f"/api/intelligent-evals/{eval_id}/config-snapshots/{foreign['id']}")
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
class TestExpiredSessionMarkdownAnnotation:
|
||||||
|
"""ADR-0011:expired 会话在 Markdown 报告尾部标注不完整证据。"""
|
||||||
|
|
||||||
|
async def test_markdown_annotates_expired_sessions(self, client, seeded_db):
|
||||||
|
eval_id = await _create_executing_eval(client)
|
||||||
|
session = await _create_session(client, eval_id)
|
||||||
|
|
||||||
|
row = seeded_db.exec(select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.id == session["id"])).one()
|
||||||
|
row.status = "expired"
|
||||||
|
row.turn_count = 3
|
||||||
|
row.closed_at = utc_now()
|
||||||
|
seeded_db.add(row)
|
||||||
|
seeded_db.commit()
|
||||||
|
|
||||||
|
await _complete_eval(client, eval_id)
|
||||||
|
|
||||||
|
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
text = resp.text
|
||||||
|
assert "## 不完整证据会话" in text
|
||||||
|
assert f"- 会话 `{session['id']}`(3 轮):60 分钟无新轮次过期,证据不完整" in text
|
||||||
|
|
||||||
|
async def test_markdown_without_expired_sessions_has_no_annotation(self, client, seeded_db):
|
||||||
|
eval_id = await _create_executing_eval(client)
|
||||||
|
await _complete_eval(client, eval_id)
|
||||||
|
|
||||||
|
resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "不完整证据会话" not in resp.text
|
||||||
Loading…
Reference in New Issue
Block a user