1317552701
Introduce 智能评估 as an evaluation paradigm parallel to static evaluation, driven by OpenClaw. The platform supplies storage, lifecycle, and reporting; OpenClaw plans and executes. - Data model: IntelligentEval + Session + Message tables (new, not reusing exploration) - Lifecycle state machine: draft → planning → pending_approval → executing → completed/cancelled/failed - Session API: create/message (channel-forwarded)/close with turn accounting - Report API: pydantic-validated structured report, executing → completed, Markdown export (pure renderer) - Alembic migration for the three tables; domain glossary added to CONTEXT.md
245 lines
8.7 KiB
Python
245 lines
8.7 KiB
Python
"""API routes for intelligent evaluation (智能评估).
|
||
|
||
领域逻辑（状态机）在 intelligent_eval/lifecycle.py；本层只做 HTTP 翻译：
|
||
NotFound→404、TransitionError→409。
|
||
"""
|
||
|
||
from typing import Any
|
||
|
||
from fastapi import APIRouter, Depends, HTTPException
|
||
from fastapi.responses import PlainTextResponse
|
||
from pydantic import BaseModel, Field
|
||
from sqlmodel import Session
|
||
|
||
from agenteval.intelligent_eval import lifecycle
|
||
from agenteval.intelligent_eval.lifecycle import (
|
||
IntelligentEvalChannelError,
|
||
IntelligentEvalNotFoundError,
|
||
IntelligentEvalTransitionError,
|
||
)
|
||
from agenteval.intelligent_eval.report import ReportModel, render_report_markdown
|
||
from agenteval.intelligent_eval.repository import IntelligentEvalSessionRepository
|
||
from agenteval.web.deps import get_db
|
||
|
||
router = APIRouter()
|
||
|
||
|
||
class CreateEvalRequest(BaseModel):
|
||
name: str = Field(min_length=1)
|
||
target_id: str = Field(min_length=1)
|
||
goal: str = Field(min_length=1)
|
||
seeds: dict[str, Any] = Field(default_factory=dict)
|
||
intent: str = ""
|
||
role_description: str = ""
|
||
time_window_hours: int = Field(default=24, ge=1)
|
||
|
||
|
||
class SubmitPlanRequest(BaseModel):
|
||
plan: dict[str, Any]
|
||
|
||
|
||
class RejectRequest(BaseModel):
|
||
feedback: str = Field(min_length=1)
|
||
|
||
|
||
class CreateSessionRequest(BaseModel):
|
||
persona: dict[str, Any] = Field(default_factory=dict)
|
||
goal: str = Field(min_length=1)
|
||
dimension: str | None = None
|
||
|
||
|
||
class SendMessageRequest(BaseModel):
|
||
content: str = Field(min_length=1)
|
||
|
||
|
||
class CloseSessionRequest(BaseModel):
|
||
verdict: dict[str, Any]
|
||
|
||
|
||
class SubmitReportRequest(BaseModel):
|
||
report: ReportModel
|
||
|
||
|
||
def _translate(exc: Exception) -> HTTPException:
|
||
if isinstance(exc, IntelligentEvalNotFoundError):
|
||
return HTTPException(status_code=404, detail=str(exc))
|
||
if isinstance(exc, IntelligentEvalChannelError):
|
||
return HTTPException(status_code=502, detail=str(exc))
|
||
return HTTPException(status_code=409, detail=exc.reason)
|
||
|
||
|
||
def _eval_response(ev, session: Session) -> dict:
|
||
data = ev.model_dump(mode="json")
|
||
sessions = IntelligentEvalSessionRepository(session).list_by_eval(ev.id)
|
||
data["session_count"] = len(sessions)
|
||
data["completed_sessions"] = sum(1 for s in sessions if s.status.value == "completed")
|
||
return data
|
||
|
||
|
||
@router.post("")
|
||
async def create_eval(request: CreateEvalRequest, session: Session = Depends(get_db)) -> dict:
|
||
ev = lifecycle.create_eval(
|
||
session,
|
||
name=request.name,
|
||
target_id=request.target_id,
|
||
goal=request.goal,
|
||
seeds=request.seeds,
|
||
intent=request.intent,
|
||
role_description=request.role_description,
|
||
time_window_hours=request.time_window_hours,
|
||
)
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.get("")
|
||
async def list_evals(session: Session = Depends(get_db)) -> dict:
|
||
evals = lifecycle.list_evals(session)
|
||
return {"intelligent_evals": [_eval_response(ev, session) for ev in evals]}
|
||
|
||
|
||
@router.get("/{eval_id}")
|
||
async def get_eval(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.get_eval(session, eval_id)
|
||
except IntelligentEvalNotFoundError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.put("/{eval_id}/plan")
|
||
async def submit_plan(eval_id: str, request: SubmitPlanRequest, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.submit_plan(session, eval_id, request.plan)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.post("/{eval_id}/approve")
|
||
async def approve(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.approve(session, eval_id)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.post("/{eval_id}/reject")
|
||
async def reject(eval_id: str, request: RejectRequest, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.reject(session, eval_id, request.feedback)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.post("/{eval_id}/cancel")
|
||
async def cancel(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.cancel(session, eval_id)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.put("/{eval_id}/report")
|
||
async def submit_report(eval_id: str, request: SubmitReportRequest, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.submit_report(session, eval_id, request.report.model_dump())
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.get("/{eval_id}/report")
|
||
async def get_report(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.get_eval(session, eval_id)
|
||
except IntelligentEvalNotFoundError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||
if ev.report is None:
|
||
raise HTTPException(status_code=404, detail="report not submitted yet")
|
||
return ev.report
|
||
|
||
|
||
@router.get("/{eval_id}/report/markdown", response_class=PlainTextResponse)
|
||
async def get_report_markdown(eval_id: str, session: Session = Depends(get_db)) -> PlainTextResponse:
|
||
try:
|
||
ev = lifecycle.get_eval(session, eval_id)
|
||
except IntelligentEvalNotFoundError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||
if ev.report is None:
|
||
raise HTTPException(status_code=404, detail="report not submitted yet")
|
||
markdown = render_report_markdown(ev.report, name=ev.name, eval_id=ev.id)
|
||
return PlainTextResponse(markdown, media_type="text/markdown; charset=utf-8")
|
||
|
||
|
||
@router.post("/{eval_id}/sessions")
|
||
async def create_session(
|
||
eval_id: str, request: CreateSessionRequest, session: Session = Depends(get_db)
|
||
) -> dict:
|
||
try:
|
||
obj = lifecycle.open_session(
|
||
session,
|
||
eval_id=eval_id,
|
||
persona=request.persona,
|
||
goal=request.goal,
|
||
dimension=request.dimension,
|
||
)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return obj.model_dump(mode="json")
|
||
|
||
|
||
@router.get("/{eval_id}/sessions")
|
||
async def list_sessions(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
sessions = lifecycle.list_sessions(session, eval_id)
|
||
except IntelligentEvalNotFoundError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||
return {"sessions": [s.model_dump(mode="json") for s in sessions]}
|
||
|
||
|
||
def _get_owned_session(eval_id: str, session_id: str, session: Session):
|
||
obj = lifecycle.get_session_by_id(session, session_id)
|
||
if obj.eval_id != eval_id:
|
||
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
|
||
return obj
|
||
|
||
|
||
@router.post("/{eval_id}/sessions/{session_id}/messages")
|
||
async def send_message(
|
||
eval_id: str, session_id: str, request: SendMessageRequest, session: Session = Depends(get_db)
|
||
) -> dict:
|
||
try:
|
||
_get_owned_session(eval_id, session_id, session)
|
||
return await lifecycle.conduct_turn(session, session_id=session_id, content=request.content)
|
||
except (
|
||
IntelligentEvalNotFoundError,
|
||
IntelligentEvalTransitionError,
|
||
IntelligentEvalChannelError,
|
||
) as exc:
|
||
raise _translate(exc) from exc
|
||
|
||
|
||
@router.post("/{eval_id}/sessions/{session_id}/close")
|
||
async def close_session(
|
||
eval_id: str, session_id: str, request: CloseSessionRequest, session: Session = Depends(get_db)
|
||
) -> dict:
|
||
try:
|
||
_get_owned_session(eval_id, session_id, session)
|
||
obj = lifecycle.close_session(session, session_id=session_id, verdict=request.verdict)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return obj.model_dump(mode="json")
|
||
|
||
|
||
@router.get("/{eval_id}/sessions/{session_id}/messages")
|
||
async def list_messages(eval_id: str, session_id: str, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
_get_owned_session(eval_id, session_id, session)
|
||
messages = lifecycle.list_messages(session, session_id)
|
||
except IntelligentEvalNotFoundError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||
return {"messages": [m.model_dump(mode="json") for m in messages]}
|