e6f98aaa6d
- Add config_snapshot.py with save/list/get/compare functions - Auto-save snapshots on eval creation and plan submission - Implement snapshot query APIs (list, get single) - Implement snapshot comparison API (diff two snapshots) - Add 8 unit tests and 7 integration tests Snapshots track config changes over time (created/plan_submitted/config_updated). All 813 tests passing.
355 lines
12 KiB
Python
355 lines
12 KiB
Python
"""Intelligent eval lifecycle (状态机 + 领域操作).
|
||
|
||
状态机：
|
||
draft → planning → pending_approval → executing → completed
|
||
→ cancelled
|
||
→ failed
|
||
pending_approval 可打回 → planning（附反馈）
|
||
executing 可取消 → cancelled
|
||
|
||
非法转换抛 IntelligentEvalTransitionError，路由层映射为 409。
|
||
"""
|
||
|
||
from typing import Any
|
||
|
||
from sqlmodel import Session
|
||
|
||
from agenteval.channels.base import ExchangeStatus, SendResult
|
||
from agenteval.channels.factory import ChannelFactory
|
||
from agenteval.config import get_settings
|
||
from agenteval.intelligent_eval.models import (
|
||
IntelligentEval,
|
||
IntelligentEvalMessage,
|
||
IntelligentEvalSession,
|
||
IntelligentEvalSessionStatus,
|
||
IntelligentEvalStatus,
|
||
)
|
||
from agenteval.intelligent_eval.repository import (
|
||
CompareAndSetStatus,
|
||
IntelligentEvalMessageRepository,
|
||
IntelligentEvalRepository,
|
||
IntelligentEvalSessionRepository,
|
||
)
|
||
from agenteval.storage.db import utc_now
|
||
from agenteval.storage.repository import TargetRepository
|
||
|
||
# 合法转换表：当前状态 → 允许的目标状态集合
|
||
_TRANSITIONS: dict[IntelligentEvalStatus, set[IntelligentEvalStatus]] = {
|
||
IntelligentEvalStatus.DRAFT: {IntelligentEvalStatus.PLANNING},
|
||
IntelligentEvalStatus.PLANNING: {IntelligentEvalStatus.PENDING_APPROVAL},
|
||
IntelligentEvalStatus.PENDING_APPROVAL: {
|
||
IntelligentEvalStatus.EXECUTING,
|
||
IntelligentEvalStatus.PLANNING,
|
||
IntelligentEvalStatus.CANCELLED,
|
||
},
|
||
IntelligentEvalStatus.EXECUTING: {
|
||
IntelligentEvalStatus.COMPLETED,
|
||
IntelligentEvalStatus.CANCELLED,
|
||
IntelligentEvalStatus.FAILED,
|
||
},
|
||
IntelligentEvalStatus.COMPLETED: set(),
|
||
IntelligentEvalStatus.CANCELLED: set(),
|
||
IntelligentEvalStatus.FAILED: set(),
|
||
}
|
||
|
||
|
||
class IntelligentEvalNotFoundError(Exception):
|
||
pass
|
||
|
||
|
||
class IntelligentEvalTransitionError(Exception):
|
||
def __init__(self, reason: str):
|
||
self.reason = reason
|
||
super().__init__(reason)
|
||
|
||
|
||
class IntelligentEvalChannelError(Exception):
|
||
pass
|
||
|
||
|
||
def _get_or_raise(repo: IntelligentEvalRepository, eval_id: str) -> IntelligentEval:
|
||
ev = repo.get(eval_id)
|
||
if ev is None:
|
||
raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found")
|
||
return ev
|
||
|
||
|
||
def _resolve_write(
|
||
eval_id: str,
|
||
result,
|
||
*,
|
||
expected: IntelligentEvalStatus,
|
||
target: IntelligentEvalStatus,
|
||
) -> IntelligentEval:
|
||
if result.status is CompareAndSetStatus.NOT_FOUND:
|
||
raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found")
|
||
if result.status is CompareAndSetStatus.CONFLICT:
|
||
raise IntelligentEvalTransitionError(
|
||
f"cannot transition from {expected.value} to {target.value}; state changed concurrently"
|
||
)
|
||
if result.evaluation is None:
|
||
raise RuntimeError(f"lifecycle write returned no evaluation: {eval_id}")
|
||
return result.evaluation
|
||
|
||
|
||
def _transition(repo: IntelligentEvalRepository, ev: IntelligentEval, target: IntelligentEvalStatus) -> IntelligentEval:
|
||
allowed = _TRANSITIONS.get(ev.status, set())
|
||
if target not in allowed:
|
||
raise IntelligentEvalTransitionError(f"cannot transition from {ev.status.value} to {target.value}")
|
||
result = repo._compare_and_set_status(
|
||
ev.id,
|
||
expected_status=ev.status,
|
||
new_status=target,
|
||
)
|
||
return _resolve_write(ev.id, result, expected=ev.status, target=target)
|
||
|
||
|
||
def create_eval(
|
||
session: Session,
|
||
*,
|
||
name: str,
|
||
target_id: str,
|
||
goal: str,
|
||
seeds: dict[str, Any],
|
||
intent: str,
|
||
role_description: str,
|
||
time_window_hours: int = 24,
|
||
) -> IntelligentEval:
|
||
"""创建智能评估并直接进入 planning 状态（draft → planning 一步完成）。"""
|
||
from agenteval.intelligent_eval.config_snapshot import save_snapshot
|
||
from agenteval.storage.db import IntelligentEvalDB
|
||
|
||
if TargetRepository(session).get(target_id) is None:
|
||
raise IntelligentEvalNotFoundError(f"target {target_id} not found")
|
||
|
||
repo = IntelligentEvalRepository(session)
|
||
ev = repo._create(
|
||
IntelligentEval(
|
||
name=name,
|
||
target_id=target_id,
|
||
status=IntelligentEvalStatus.PLANNING,
|
||
goal=goal,
|
||
seeds=seeds,
|
||
intent=intent,
|
||
role_description=role_description,
|
||
time_window_hours=time_window_hours,
|
||
created_at=utc_now(),
|
||
updated_at=utc_now(),
|
||
)
|
||
)
|
||
|
||
# Save config snapshot
|
||
eval_db = session.get(IntelligentEvalDB, ev.id)
|
||
if eval_db:
|
||
save_snapshot(eval_db, "created", "user", session)
|
||
|
||
return ev
|
||
|
||
|
||
def submit_plan(session: Session, eval_id: str, plan: dict[str, Any]) -> IntelligentEval:
|
||
"""OpenClaw 提交粗计划：planning → pending_approval。"""
|
||
from agenteval.intelligent_eval.config_snapshot import save_snapshot
|
||
from agenteval.storage.db import IntelligentEvalDB
|
||
|
||
repo = IntelligentEvalRepository(session)
|
||
result = repo._submit_plan_if_planning(eval_id, plan)
|
||
ev = _resolve_write(
|
||
eval_id,
|
||
result,
|
||
expected=IntelligentEvalStatus.PLANNING,
|
||
target=IntelligentEvalStatus.PENDING_APPROVAL,
|
||
)
|
||
|
||
# Save config snapshot
|
||
eval_db = session.get(IntelligentEvalDB, eval_id)
|
||
if eval_db:
|
||
save_snapshot(eval_db, "plan_submitted", "openclaw", session)
|
||
|
||
return ev
|
||
|
||
|
||
def approve(session: Session, eval_id: str) -> IntelligentEval:
|
||
"""用户批准：pending_approval → executing。"""
|
||
repo = IntelligentEvalRepository(session)
|
||
ev = _get_or_raise(repo, eval_id)
|
||
return _transition(repo, ev, IntelligentEvalStatus.EXECUTING)
|
||
|
||
|
||
def reject(session: Session, eval_id: str, feedback: str) -> IntelligentEval:
|
||
"""用户打回：pending_approval → planning（附反馈）。"""
|
||
repo = IntelligentEvalRepository(session)
|
||
result = repo._reject_plan_if_pending(eval_id, feedback)
|
||
return _resolve_write(
|
||
eval_id,
|
||
result,
|
||
expected=IntelligentEvalStatus.PENDING_APPROVAL,
|
||
target=IntelligentEvalStatus.PLANNING,
|
||
)
|
||
|
||
|
||
def cancel(session: Session, eval_id: str) -> IntelligentEval:
|
||
"""用户取消：pending_approval / executing → cancelled。"""
|
||
repo = IntelligentEvalRepository(session)
|
||
ev = _get_or_raise(repo, eval_id)
|
||
return _transition(repo, ev, IntelligentEvalStatus.CANCELLED)
|
||
|
||
|
||
def submit_report(session: Session, eval_id: str, report: dict[str, Any]) -> IntelligentEval:
|
||
"""OpenClaw 提交结构化报告：executing → completed。
|
||
|
||
报告结构校验在路由层（pydantic），此处只负责落库与状态迁移。
|
||
"""
|
||
repo = IntelligentEvalRepository(session)
|
||
result = repo._submit_report_if_executing(eval_id, report)
|
||
return _resolve_write(
|
||
eval_id,
|
||
result,
|
||
expected=IntelligentEvalStatus.EXECUTING,
|
||
target=IntelligentEvalStatus.COMPLETED,
|
||
)
|
||
|
||
|
||
def get_eval(session: Session, eval_id: str) -> IntelligentEval:
|
||
repo = IntelligentEvalRepository(session)
|
||
return _get_or_raise(repo, eval_id)
|
||
|
||
|
||
def list_evals(session: Session) -> list[IntelligentEval]:
|
||
return IntelligentEvalRepository(session).list_all()
|
||
|
||
|
||
def _get_session_or_raise(repo: IntelligentEvalSessionRepository, session_id: str) -> IntelligentEvalSession:
|
||
obj = repo.get(session_id)
|
||
if obj is None:
|
||
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
|
||
return obj
|
||
|
||
|
||
def open_session(
|
||
session: Session,
|
||
*,
|
||
eval_id: str,
|
||
persona: dict[str, Any],
|
||
goal: str,
|
||
dimension: str | None = None,
|
||
) -> IntelligentEvalSession:
|
||
"""创建虚拟用户会话：仅 executing 状态的评估可创建。"""
|
||
eval_repo = IntelligentEvalRepository(session)
|
||
ev = _get_or_raise(eval_repo, eval_id)
|
||
if ev.status != IntelligentEvalStatus.EXECUTING:
|
||
raise IntelligentEvalTransitionError(f"评估不在执行中（当前 {ev.status.value}），无法创建会话")
|
||
repo = IntelligentEvalSessionRepository(session)
|
||
status, created = repo._create_if_executing(
|
||
IntelligentEvalSession(
|
||
eval_id=ev.id,
|
||
target_id=ev.target_id,
|
||
persona=persona,
|
||
goal=goal,
|
||
dimension=dimension,
|
||
)
|
||
)
|
||
if status is CompareAndSetStatus.CONFLICT:
|
||
raise IntelligentEvalTransitionError("评估不在执行中，会话创建被拒绝")
|
||
if status is CompareAndSetStatus.NOT_FOUND or created is None:
|
||
raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found")
|
||
return created
|
||
|
||
|
||
async def conduct_turn(session: Session, *, eval_id: str, session_id: str, content: str) -> dict[str, Any]:
|
||
"""一轮完整问答：状态检查 → 通道往返 → 双条消息落库 → 轮次自增。"""
|
||
repo = IntelligentEvalSessionRepository(session)
|
||
obj = _get_owned_session_or_raise(repo, eval_id, session_id)
|
||
if obj.status != IntelligentEvalSessionStatus.RUNNING:
|
||
raise IntelligentEvalTransitionError("会话不在进行中，拒收消息")
|
||
|
||
target = TargetRepository(session).get(obj.target_id)
|
||
if not target:
|
||
raise IntelligentEvalNotFoundError("session target not found")
|
||
|
||
message_repo = IntelligentEvalMessageRepository(session)
|
||
sent_at = utc_now()
|
||
channel = ChannelFactory.create(target)
|
||
|
||
async def record_sent(_send_result: SendResult) -> None:
|
||
message = IntelligentEvalMessage(
|
||
session_id=obj.id,
|
||
role="user",
|
||
content=content,
|
||
created_at=sent_at,
|
||
)
|
||
status = message_repo._create_user_and_increment(message)
|
||
if status is CompareAndSetStatus.NOT_FOUND:
|
||
raise IntelligentEvalNotFoundError(f"intelligent eval session {obj.id} not found")
|
||
if status is CompareAndSetStatus.CONFLICT:
|
||
raise IntelligentEvalTransitionError("会话不在进行中，拒收消息")
|
||
|
||
outcome = await channel.exchange(
|
||
content,
|
||
timeout=get_settings().poll_reply_timeout,
|
||
on_sent=record_sent,
|
||
)
|
||
if outcome.status is ExchangeStatus.SEND_FAILED:
|
||
raise IntelligentEvalChannelError(f"评测对象通道发送失败: {outcome.reason}")
|
||
if outcome.status is ExchangeStatus.POLL_FAILED:
|
||
raise IntelligentEvalChannelError(f"等待评测对象回复失败: {outcome.reason}")
|
||
if outcome.status is ExchangeStatus.REPLY_TIMEOUT:
|
||
raise IntelligentEvalChannelError("等待评测对象回复超时")
|
||
|
||
received_at = utc_now()
|
||
latency_ms = (
|
||
outcome.latency_ms if outcome.latency_ms is not None else int((received_at - sent_at).total_seconds() * 1000)
|
||
)
|
||
reply_text = outcome.reply_text or ""
|
||
message_repo._create(
|
||
IntelligentEvalMessage(
|
||
session_id=obj.id,
|
||
role="assistant",
|
||
content=reply_text,
|
||
latency_ms=latency_ms,
|
||
created_at=received_at,
|
||
)
|
||
)
|
||
return {"reply": reply_text, "latency_ms": latency_ms, "turn_count": obj.turn_count + 1}
|
||
|
||
|
||
def close_session(
|
||
session: Session,
|
||
*,
|
||
eval_id: str,
|
||
session_id: str,
|
||
verdict: dict[str, Any],
|
||
) -> IntelligentEvalSession:
|
||
"""关闭会话并记录结论（verdict）；仅 running 会话可关闭。"""
|
||
repo = IntelligentEvalSessionRepository(session)
|
||
obj = _get_owned_session_or_raise(repo, eval_id, session_id)
|
||
status, closed = repo._close_if_running(obj.id, verdict)
|
||
if status is CompareAndSetStatus.NOT_FOUND:
|
||
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
|
||
if status is CompareAndSetStatus.CONFLICT:
|
||
raise IntelligentEvalTransitionError("会话不在进行中，无法关闭")
|
||
if closed is None:
|
||
raise RuntimeError(f"session close returned no session: {session_id}")
|
||
return closed
|
||
|
||
|
||
def _get_owned_session_or_raise(
|
||
repo: IntelligentEvalSessionRepository,
|
||
eval_id: str,
|
||
session_id: str,
|
||
) -> IntelligentEvalSession:
|
||
obj = _get_session_or_raise(repo, session_id)
|
||
if obj.eval_id != eval_id:
|
||
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
|
||
return obj
|
||
|
||
|
||
def list_sessions(session: Session, eval_id: str) -> list[IntelligentEvalSession]:
|
||
eval_repo = IntelligentEvalRepository(session)
|
||
_get_or_raise(eval_repo, eval_id)
|
||
return IntelligentEvalSessionRepository(session).list_by_eval(eval_id)
|
||
|
||
|
||
def list_messages(session: Session, *, eval_id: str, session_id: str) -> list[IntelligentEvalMessage]:
|
||
_get_owned_session_or_raise(IntelligentEvalSessionRepository(session), eval_id, session_id)
|
||
return IntelligentEvalMessageRepository(session).list_by_session(session_id)
|