"""Intelligent eval lifecycle (状态机 + 领域操作). 状态机: draft → planning → pending_approval → executing → completed → cancelled → failed pending_approval 可打回 → planning(附反馈) executing 可取消 → cancelled 非法转换抛 IntelligentEvalTransitionError,路由层映射为 409。 """ from typing import Any, Optional from sqlmodel import Session from agenteval.channels.base import ExchangeStatus, SendResult from agenteval.channels.factory import ChannelFactory from agenteval.config import get_settings from agenteval.intelligent_eval.models import ( IntelligentEval, IntelligentEvalMessage, IntelligentEvalSession, IntelligentEvalSessionStatus, IntelligentEvalStatus, ) from agenteval.intelligent_eval.repository import ( CompareAndSetStatus, IntelligentEvalMessageRepository, IntelligentEvalRepository, IntelligentEvalSessionRepository, ) from agenteval.storage.db import utc_now from agenteval.storage.repository import TargetRepository # 合法转换表:当前状态 → 允许的目标状态集合 _TRANSITIONS: dict[IntelligentEvalStatus, set[IntelligentEvalStatus]] = { IntelligentEvalStatus.DRAFT: {IntelligentEvalStatus.PLANNING}, IntelligentEvalStatus.PLANNING: {IntelligentEvalStatus.PENDING_APPROVAL}, IntelligentEvalStatus.PENDING_APPROVAL: { IntelligentEvalStatus.EXECUTING, IntelligentEvalStatus.PLANNING, IntelligentEvalStatus.CANCELLED, }, IntelligentEvalStatus.EXECUTING: { IntelligentEvalStatus.COMPLETED, IntelligentEvalStatus.CANCELLED, IntelligentEvalStatus.FAILED, }, IntelligentEvalStatus.COMPLETED: set(), IntelligentEvalStatus.CANCELLED: set(), IntelligentEvalStatus.FAILED: set(), } class IntelligentEvalNotFoundError(Exception): pass class IntelligentEvalTransitionError(Exception): def __init__(self, reason: str): self.reason = reason super().__init__(reason) class IntelligentEvalChannelError(Exception): pass def _get_or_raise(repo: IntelligentEvalRepository, eval_id: str) -> IntelligentEval: ev = repo.get(eval_id) if ev is None: raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found") return ev def _resolve_write( eval_id: str, result, *, expected: IntelligentEvalStatus, target: IntelligentEvalStatus, ) -> IntelligentEval: if result.status is CompareAndSetStatus.NOT_FOUND: raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found") if result.status is CompareAndSetStatus.CONFLICT: raise IntelligentEvalTransitionError( f"cannot transition from {expected.value} to {target.value}; state changed concurrently" ) if result.evaluation is None: raise RuntimeError(f"lifecycle write returned no evaluation: {eval_id}") return result.evaluation def _transition(repo: IntelligentEvalRepository, ev: IntelligentEval, target: IntelligentEvalStatus) -> IntelligentEval: allowed = _TRANSITIONS.get(ev.status, set()) if target not in allowed: raise IntelligentEvalTransitionError(f"cannot transition from {ev.status.value} to {target.value}") result = repo._compare_and_set_status( ev.id, expected_status=ev.status, new_status=target, ) return _resolve_write(ev.id, result, expected=ev.status, target=target) def create_eval( session: Session, *, name: str, target_id: str, goal: str, seeds: dict[str, Any], intent: str, role_description: str, time_window_hours: int = 24, ) -> IntelligentEval: """创建智能评估并直接进入 planning 状态(draft → planning 一步完成)。""" from agenteval.intelligent_eval.config_snapshot import save_snapshot from agenteval.storage.db import IntelligentEvalDB if TargetRepository(session).get(target_id) is None: raise IntelligentEvalNotFoundError(f"target {target_id} not found") repo = IntelligentEvalRepository(session) ev = repo._create( IntelligentEval( name=name, target_id=target_id, status=IntelligentEvalStatus.PLANNING, goal=goal, seeds=seeds, intent=intent, role_description=role_description, time_window_hours=time_window_hours, created_at=utc_now(), updated_at=utc_now(), ) ) # Save config snapshot eval_db = session.get(IntelligentEvalDB, ev.id) if eval_db: save_snapshot(eval_db, "created", "user", session) return ev def submit_plan(session: Session, eval_id: str, plan: dict[str, Any]) -> IntelligentEval: """OpenClaw 提交粗计划:planning → pending_approval。""" from agenteval.intelligent_eval.config_snapshot import save_snapshot from agenteval.storage.db import IntelligentEvalDB repo = IntelligentEvalRepository(session) result = repo._submit_plan_if_planning(eval_id, plan) ev = _resolve_write( eval_id, result, expected=IntelligentEvalStatus.PLANNING, target=IntelligentEvalStatus.PENDING_APPROVAL, ) # Save config snapshot eval_db = session.get(IntelligentEvalDB, eval_id) if eval_db: save_snapshot(eval_db, "plan_submitted", "openclaw", session) return ev def approve(session: Session, eval_id: str) -> IntelligentEval: """用户批准:pending_approval → executing。""" repo = IntelligentEvalRepository(session) ev = _get_or_raise(repo, eval_id) return _transition(repo, ev, IntelligentEvalStatus.EXECUTING) def reject(session: Session, eval_id: str, feedback: str) -> IntelligentEval: """用户打回:pending_approval → planning(附反馈)。""" repo = IntelligentEvalRepository(session) result = repo._reject_plan_if_pending(eval_id, feedback) return _resolve_write( eval_id, result, expected=IntelligentEvalStatus.PENDING_APPROVAL, target=IntelligentEvalStatus.PLANNING, ) def cancel(session: Session, eval_id: str) -> IntelligentEval: """用户取消:pending_approval / executing → cancelled。""" repo = IntelligentEvalRepository(session) ev = _get_or_raise(repo, eval_id) return _transition(repo, ev, IntelligentEvalStatus.CANCELLED) def submit_report(session: Session, eval_id: str, report: dict[str, Any]) -> IntelligentEval: """OpenClaw 提交结构化报告:executing → completed。 报告结构校验在路由层(pydantic),此处只负责落库与状态迁移。 """ repo = IntelligentEvalRepository(session) result = repo._submit_report_if_executing(eval_id, report) return _resolve_write( eval_id, result, expected=IntelligentEvalStatus.EXECUTING, target=IntelligentEvalStatus.COMPLETED, ) def get_eval(session: Session, eval_id: str) -> IntelligentEval: repo = IntelligentEvalRepository(session) return _get_or_raise(repo, eval_id) def list_evals(session: Session) -> list[IntelligentEval]: return IntelligentEvalRepository(session).list_all() def list_evals_page( session: Session, offset: int, limit: int, status: Optional[str] = None ) -> tuple[list[IntelligentEval], int, dict[str, int]]: """Return one page of evals (newest first), total count, and per-status counts.""" repo = IntelligentEvalRepository(session) return repo.list_page(offset, limit, status), repo.count(), repo.count_by_status() def eval_status_counts(session: Session) -> dict[str, int]: """Count evaluations per status (for the list page stat bar).""" return IntelligentEvalRepository(session).count_by_status() def _get_session_or_raise(repo: IntelligentEvalSessionRepository, session_id: str) -> IntelligentEvalSession: obj = repo.get(session_id) if obj is None: raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found") return obj def open_session( session: Session, *, eval_id: str, persona: dict[str, Any], goal: str, dimension: str | None = None, ) -> IntelligentEvalSession: """创建虚拟用户会话:仅 executing 状态的评估可创建。""" eval_repo = IntelligentEvalRepository(session) ev = _get_or_raise(eval_repo, eval_id) if ev.status != IntelligentEvalStatus.EXECUTING: raise IntelligentEvalTransitionError(f"评估不在执行中(当前 {ev.status.value}),无法创建会话") repo = IntelligentEvalSessionRepository(session) status, created = repo._create_if_executing( IntelligentEvalSession( eval_id=ev.id, target_id=ev.target_id, persona=persona, goal=goal, dimension=dimension, ) ) if status is CompareAndSetStatus.CONFLICT: raise IntelligentEvalTransitionError("评估不在执行中,会话创建被拒绝") if status is CompareAndSetStatus.NOT_FOUND or created is None: raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found") return created async def conduct_turn(session: Session, *, eval_id: str, session_id: str, content: str) -> dict[str, Any]: """一轮完整问答:状态检查 → 通道往返 → 双条消息落库 → 轮次自增。""" repo = IntelligentEvalSessionRepository(session) obj = _get_owned_session_or_raise(repo, eval_id, session_id) if obj.status != IntelligentEvalSessionStatus.RUNNING: raise IntelligentEvalTransitionError("会话不在进行中,拒收消息") target = TargetRepository(session).get(obj.target_id) if not target: raise IntelligentEvalNotFoundError("session target not found") message_repo = IntelligentEvalMessageRepository(session) sent_at = utc_now() channel = ChannelFactory.create(target) async def record_sent(_send_result: SendResult) -> None: message = IntelligentEvalMessage( session_id=obj.id, role="user", content=content, created_at=sent_at, ) status = message_repo._create_user_and_increment(message) if status is CompareAndSetStatus.NOT_FOUND: raise IntelligentEvalNotFoundError(f"intelligent eval session {obj.id} not found") if status is CompareAndSetStatus.CONFLICT: raise IntelligentEvalTransitionError("会话不在进行中,拒收消息") outcome = await channel.exchange( content, timeout=get_settings().poll_reply_timeout, on_sent=record_sent, ) if outcome.status is ExchangeStatus.SEND_FAILED: raise IntelligentEvalChannelError(f"评测对象通道发送失败: {outcome.reason}") if outcome.status is ExchangeStatus.POLL_FAILED: raise IntelligentEvalChannelError(f"等待评测对象回复失败: {outcome.reason}") if outcome.status is ExchangeStatus.REPLY_TIMEOUT: raise IntelligentEvalChannelError("等待评测对象回复超时") received_at = utc_now() latency_ms = ( outcome.latency_ms if outcome.latency_ms is not None else int((received_at - sent_at).total_seconds() * 1000) ) reply_text = outcome.reply_text or "" message_repo._create( IntelligentEvalMessage( session_id=obj.id, role="assistant", content=reply_text, latency_ms=latency_ms, created_at=received_at, ) ) return {"reply": reply_text, "latency_ms": latency_ms, "turn_count": obj.turn_count + 1} def close_session( session: Session, *, eval_id: str, session_id: str, verdict: dict[str, Any], ) -> IntelligentEvalSession: """关闭会话并记录结论(verdict);仅 running 会话可关闭。""" repo = IntelligentEvalSessionRepository(session) obj = _get_owned_session_or_raise(repo, eval_id, session_id) status, closed = repo._close_if_running(obj.id, verdict) if status is CompareAndSetStatus.NOT_FOUND: raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found") if status is CompareAndSetStatus.CONFLICT: raise IntelligentEvalTransitionError("会话不在进行中,无法关闭") if closed is None: raise RuntimeError(f"session close returned no session: {session_id}") return closed def _get_owned_session_or_raise( repo: IntelligentEvalSessionRepository, eval_id: str, session_id: str, ) -> IntelligentEvalSession: obj = _get_session_or_raise(repo, session_id) if obj.eval_id != eval_id: raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found") return obj def list_sessions(session: Session, eval_id: str) -> list[IntelligentEvalSession]: eval_repo = IntelligentEvalRepository(session) _get_or_raise(eval_repo, eval_id) return IntelligentEvalSessionRepository(session).list_by_eval(eval_id) def list_messages(session: Session, *, eval_id: str, session_id: str) -> list[IntelligentEvalMessage]: _get_owned_session_or_raise(IntelligentEvalSessionRepository(session), eval_id, session_id) return IntelligentEvalMessageRepository(session).list_by_session(session_id)