eb4944a8bd
常见故障自愈有上限，超限收敛终态且可见：任务 attempts 上限、会话过期、 planning 双闸、executing 超窗兜底、触发失败计数判死、孤儿 agent 双管、 fire-and-forget 触发；open_session 预算硬闸门、settle 按终态区分、报告 scores 归一化；cron 池遗留面全删。
478 lines
17 KiB
Python
478 lines
17 KiB
Python
"""API routes for intelligent evaluation (智能评估).
|
||
|
||
领域逻辑（状态机）在 intelligent_eval/lifecycle.py；本层只做 HTTP 翻译：
|
||
NotFound→404、TransitionError→409。
|
||
"""
|
||
|
||
from typing import Any
|
||
|
||
from fastapi import APIRouter, Depends, HTTPException
|
||
from fastapi.responses import PlainTextResponse
|
||
from pydantic import BaseModel, Field
|
||
from sqlmodel import Session
|
||
|
||
from agenteval.intelligent_eval import lifecycle
|
||
from agenteval.intelligent_eval.lifecycle import (
|
||
IntelligentEvalChannelError,
|
||
IntelligentEvalNotFoundError,
|
||
IntelligentEvalTransitionError,
|
||
)
|
||
from agenteval.intelligent_eval.models import IntelligentEvalSessionStatus
|
||
from agenteval.intelligent_eval.read_model import IntelligentEvalReadModel
|
||
from agenteval.intelligent_eval.report import ReportModel, render_report_markdown
|
||
from agenteval.intelligent_eval.repository import IntelligentEvalSessionRepository
|
||
from agenteval.web.deps import get_db
|
||
|
||
router = APIRouter()
|
||
|
||
|
||
class CreateEvalRequest(BaseModel):
|
||
name: str = Field(min_length=1)
|
||
target_id: str = Field(min_length=1)
|
||
goal: str = Field(min_length=1)
|
||
seeds: dict[str, Any] = Field(default_factory=dict)
|
||
intent: str = ""
|
||
role_description: str = ""
|
||
time_window_hours: int = Field(default=24, ge=1)
|
||
|
||
|
||
class SubmitPlanRequest(BaseModel):
|
||
plan: dict[str, Any]
|
||
|
||
|
||
class RejectRequest(BaseModel):
|
||
feedback: str = Field(min_length=1)
|
||
|
||
|
||
class CreateSessionRequest(BaseModel):
|
||
persona: dict[str, Any] = Field(default_factory=dict)
|
||
goal: str = Field(min_length=1)
|
||
dimension: str | None = None
|
||
|
||
|
||
class SendMessageRequest(BaseModel):
|
||
content: str = Field(min_length=1)
|
||
|
||
|
||
class CloseSessionRequest(BaseModel):
|
||
verdict: dict[str, Any]
|
||
|
||
|
||
class SubmitReportRequest(BaseModel):
|
||
report: ReportModel
|
||
|
||
|
||
def _translate(exc: Exception) -> HTTPException:
|
||
if isinstance(exc, IntelligentEvalNotFoundError):
|
||
return HTTPException(status_code=404, detail=str(exc))
|
||
if isinstance(exc, IntelligentEvalChannelError):
|
||
return HTTPException(status_code=502, detail=str(exc))
|
||
return HTTPException(status_code=409, detail=exc.reason)
|
||
|
||
|
||
def _eval_response(ev, session: Session) -> dict:
|
||
"""Serialize one stable intelligent-evaluation read projection."""
|
||
|
||
projection = IntelligentEvalReadModel(session).list_item(ev)
|
||
return projection.model_dump(mode="json")
|
||
|
||
|
||
@router.post("")
|
||
async def create_eval(request: CreateEvalRequest, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.create_eval(
|
||
session,
|
||
name=request.name,
|
||
target_id=request.target_id,
|
||
goal=request.goal,
|
||
seeds=request.seeds,
|
||
intent=request.intent,
|
||
role_description=request.role_description,
|
||
time_window_hours=request.time_window_hours,
|
||
)
|
||
except IntelligentEvalNotFoundError as exc:
|
||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.get("")
|
||
async def list_evals(
|
||
page: int | None = None,
|
||
page_size: int = 20,
|
||
status: str | None = None,
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
"""List intelligent evaluations, optionally paginated and filtered by status.
|
||
|
||
不传 ``page`` 时返回全部（向后兼容）；传 ``page``（从 1 起）时按
|
||
``created_at`` 倒序分页，返回 ``total`` + ``stats``（各状态计数）供前端
|
||
服务端分页与状态统计条。
|
||
"""
|
||
reader = IntelligentEvalReadModel(session)
|
||
if page is None:
|
||
evals = lifecycle.list_evals(session)
|
||
return {"intelligent_evals": [item.model_dump(mode="json") for item in reader.list_items(evals)]}
|
||
page_size = max(1, min(page_size, 100))
|
||
offset = (max(1, page) - 1) * page_size
|
||
evals, total, stats = lifecycle.list_evals_page(session, offset, page_size, status)
|
||
return {
|
||
"intelligent_evals": [item.model_dump(mode="json") for item in reader.list_items(evals)],
|
||
"total": total,
|
||
"stats": stats,
|
||
"page": page,
|
||
"page_size": page_size,
|
||
}
|
||
|
||
|
||
@router.get("/tasks")
|
||
async def list_tasks(
|
||
status: str | None = None,
|
||
limit: int = 100,
|
||
eval_id: str | None = None,
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
"""List task-queue entries with eval names (monitor UI).
|
||
|
||
注意：此端点必须注册在 ``/{eval_id}`` 之前，否则 ``/tasks`` 会被
|
||
``{eval_id}`` 捕获为 eval_id="tasks"。
|
||
"""
|
||
from agenteval.intelligent_eval.task_queue import list_tasks as _list
|
||
|
||
return _list(session, status=status, limit=limit, eval_id=eval_id)
|
||
|
||
|
||
@router.get("/{eval_id}")
|
||
async def get_eval(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
projection = IntelligentEvalReadModel(session).detail_by_id(eval_id)
|
||
if projection is None:
|
||
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
|
||
return projection.model_dump(mode="json")
|
||
|
||
|
||
@router.get("/{eval_id}/execution-progress")
|
||
async def get_execution_progress(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
"""执行过程视图：生命周期阶段、阻塞点、下一步动作与时段计划 vs 实际。"""
|
||
|
||
progress = IntelligentEvalReadModel(session).execution_progress_by_id(eval_id)
|
||
if progress is None:
|
||
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
|
||
return progress.model_dump(mode="json")
|
||
|
||
|
||
@router.put("/{eval_id}/plan")
|
||
async def submit_plan(eval_id: str, request: SubmitPlanRequest, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.submit_plan(session, eval_id, request.plan)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.post("/{eval_id}/approve")
|
||
async def approve(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.approve(session, eval_id)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.post("/{eval_id}/reject")
|
||
async def reject(eval_id: str, request: RejectRequest, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.reject(session, eval_id, request.feedback)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.post("/{eval_id}/cancel")
|
||
async def cancel(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.cancel(session, eval_id)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.put("/{eval_id}/report")
|
||
async def submit_report(eval_id: str, request: SubmitReportRequest, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
ev = lifecycle.submit_report(session, eval_id, request.report.model_dump())
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return _eval_response(ev, session)
|
||
|
||
|
||
@router.get("/{eval_id}/report")
|
||
async def get_report(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
report = IntelligentEvalReadModel(session).report_by_eval(eval_id)
|
||
if report is None:
|
||
raise HTTPException(status_code=404, detail="report not submitted yet")
|
||
return report[1]
|
||
|
||
|
||
@router.get("/{eval_id}/report/markdown", response_class=PlainTextResponse)
|
||
async def get_report_markdown(eval_id: str, session: Session = Depends(get_db)) -> PlainTextResponse:
|
||
report = IntelligentEvalReadModel(session).report_by_eval(eval_id)
|
||
if report is None:
|
||
raise HTTPException(status_code=404, detail="report not submitted yet")
|
||
name, payload = report
|
||
markdown = render_report_markdown(payload, name=name, eval_id=eval_id)
|
||
# ADR-0011：expired 会话按不完整证据标注，提醒读者结论证据不全
|
||
expired_sessions = [
|
||
s
|
||
for s in IntelligentEvalSessionRepository(session).list_by_eval(eval_id)
|
||
if s.status == IntelligentEvalSessionStatus.EXPIRED
|
||
]
|
||
if expired_sessions:
|
||
lines = ["", "## 不完整证据会话", ""]
|
||
for s in expired_sessions:
|
||
lines.append(f"- 会话 `{s.id}`（{s.turn_count} 轮）：60 分钟无新轮次过期，证据不完整")
|
||
markdown = markdown.rstrip() + "\n" + "\n".join(lines) + "\n"
|
||
return PlainTextResponse(markdown, media_type="text/markdown; charset=utf-8")
|
||
|
||
|
||
@router.post("/{eval_id}/sessions")
|
||
async def create_session(eval_id: str, request: CreateSessionRequest, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
obj = lifecycle.open_session(
|
||
session,
|
||
eval_id=eval_id,
|
||
persona=request.persona,
|
||
goal=request.goal,
|
||
dimension=request.dimension,
|
||
)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return obj.model_dump(mode="json")
|
||
|
||
|
||
@router.get("/{eval_id}/sessions")
|
||
async def list_sessions(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
sessions = IntelligentEvalReadModel(session).sessions_by_eval(eval_id)
|
||
if sessions is None:
|
||
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
|
||
return {"sessions": [s.model_dump(mode="json") for s in sessions]}
|
||
|
||
|
||
@router.post("/{eval_id}/sessions/{session_id}/messages")
|
||
async def send_message(
|
||
eval_id: str, session_id: str, request: SendMessageRequest, session: Session = Depends(get_db)
|
||
) -> dict:
|
||
try:
|
||
return await lifecycle.conduct_turn(
|
||
session,
|
||
eval_id=eval_id,
|
||
session_id=session_id,
|
||
content=request.content,
|
||
)
|
||
except (
|
||
IntelligentEvalNotFoundError,
|
||
IntelligentEvalTransitionError,
|
||
IntelligentEvalChannelError,
|
||
) as exc:
|
||
raise _translate(exc) from exc
|
||
|
||
|
||
@router.post("/{eval_id}/sessions/{session_id}/close")
|
||
async def close_session(
|
||
eval_id: str, session_id: str, request: CloseSessionRequest, session: Session = Depends(get_db)
|
||
) -> dict:
|
||
try:
|
||
obj = lifecycle.close_session(
|
||
session,
|
||
eval_id=eval_id,
|
||
session_id=session_id,
|
||
verdict=request.verdict,
|
||
)
|
||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||
raise _translate(exc) from exc
|
||
return obj.model_dump(mode="json")
|
||
|
||
|
||
@router.get("/{eval_id}/sessions/{session_id}/messages")
|
||
async def list_messages(eval_id: str, session_id: str, session: Session = Depends(get_db)) -> dict:
|
||
messages = IntelligentEvalReadModel(session).messages_by_session(eval_id, session_id)
|
||
if messages is None:
|
||
raise HTTPException(status_code=404, detail=f"intelligent eval session {session_id} not found")
|
||
return {"messages": [m.model_dump(mode="json") for m in messages]}
|
||
|
||
|
||
@router.get("/tasks/next")
|
||
async def get_next_task(session: Session = Depends(get_db)) -> dict:
|
||
"""Get next pending task for OpenClaw workers."""
|
||
from agenteval.intelligent_eval.task_queue import get_next_task_with_eval
|
||
|
||
result = get_next_task_with_eval(session)
|
||
return result if result is not None else {"task": None}
|
||
|
||
|
||
@router.post("/tasks/{task_id}/assign")
|
||
async def assign_task(task_id: str, cron_id: str, session: Session = Depends(get_db)) -> dict:
|
||
"""Assign a task to a cron worker."""
|
||
from agenteval.intelligent_eval import task_queue
|
||
|
||
success = task_queue.assign_task(task_id, cron_id, session)
|
||
if not success:
|
||
raise HTTPException(status_code=404, detail="task not found or already assigned")
|
||
return {"success": True}
|
||
|
||
|
||
@router.post("/tasks/{task_id}/complete")
|
||
async def complete_task(
|
||
task_id: str,
|
||
success: bool,
|
||
error: str | None = None,
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
"""Mark a task as completed or failed."""
|
||
from agenteval.intelligent_eval import task_queue
|
||
|
||
completed = task_queue.complete_task(task_id, success, error, session)
|
||
if not completed:
|
||
raise HTTPException(status_code=404, detail="task not found")
|
||
return {"success": True}
|
||
|
||
|
||
class DecisionLogRequest(BaseModel):
|
||
decision_type: str = Field(min_length=1) # execute_session / wait / start_analysis
|
||
reason: str = Field(min_length=1)
|
||
context: dict[str, Any] = Field(default_factory=dict)
|
||
cron_id: str = Field(min_length=1)
|
||
|
||
|
||
@router.post("/{eval_id}/decision-logs")
|
||
async def create_decision_log(
|
||
eval_id: str,
|
||
request: DecisionLogRequest,
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
"""Create a decision log entry for an intelligent eval."""
|
||
from agenteval.intelligent_eval.decision_logs import create_decision_log as _create
|
||
|
||
try:
|
||
return _create(eval_id, request.decision_type, request.reason, request.cron_id, request.context, session)
|
||
except LookupError as e:
|
||
raise HTTPException(status_code=404, detail=str(e)) from e
|
||
|
||
|
||
@router.get("/{eval_id}/decision-logs")
|
||
async def list_decision_logs(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
"""List all decision logs for an evaluation."""
|
||
from agenteval.intelligent_eval.decision_logs import list_decision_logs as _list
|
||
|
||
try:
|
||
return {"logs": _list(eval_id, session)}
|
||
except LookupError as e:
|
||
raise HTTPException(status_code=404, detail=str(e)) from e
|
||
|
||
|
||
@router.get("/{eval_id}/config-snapshots")
|
||
async def list_config_snapshots(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||
"""List all config snapshots for an evaluation."""
|
||
from agenteval.intelligent_eval import config_snapshot
|
||
from agenteval.storage.db import IntelligentEvalDB
|
||
|
||
# Verify eval exists
|
||
eval_db = session.get(IntelligentEvalDB, eval_id)
|
||
if eval_db is None:
|
||
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
|
||
|
||
snapshots = config_snapshot.list_snapshots(eval_id, session)
|
||
return {
|
||
"snapshots": [
|
||
{
|
||
"id": s.id,
|
||
"eval_id": s.eval_id,
|
||
"snapshot_type": s.snapshot_type,
|
||
"goal": s.goal,
|
||
"seeds": s.get_seeds(),
|
||
"intent": s.intent,
|
||
"role_description": s.role_description,
|
||
"time_window_hours": s.time_window_hours,
|
||
"plan": s.get_plan(),
|
||
"created_at": s.created_at.isoformat() if s.created_at else None,
|
||
"created_by": s.created_by,
|
||
}
|
||
for s in snapshots
|
||
]
|
||
}
|
||
|
||
|
||
@router.get("/{eval_id}/config-snapshots/{snapshot_id}")
|
||
async def get_config_snapshot(eval_id: str, snapshot_id: str, session: Session = Depends(get_db)) -> dict:
|
||
"""Get a single config snapshot."""
|
||
from agenteval.intelligent_eval import config_snapshot
|
||
from agenteval.storage.db import IntelligentEvalDB
|
||
|
||
# Verify eval exists
|
||
eval_db = session.get(IntelligentEvalDB, eval_id)
|
||
if eval_db is None:
|
||
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
|
||
|
||
snapshot = config_snapshot.get_snapshot(snapshot_id, session)
|
||
if snapshot is None or snapshot.eval_id != eval_id:
|
||
raise HTTPException(status_code=404, detail=f"snapshot {snapshot_id} not found")
|
||
|
||
return {
|
||
"id": snapshot.id,
|
||
"eval_id": snapshot.eval_id,
|
||
"snapshot_type": snapshot.snapshot_type,
|
||
"goal": snapshot.goal,
|
||
"seeds": snapshot.get_seeds(),
|
||
"intent": snapshot.intent,
|
||
"role_description": snapshot.role_description,
|
||
"time_window_hours": snapshot.time_window_hours,
|
||
"plan": snapshot.get_plan(),
|
||
"created_at": snapshot.created_at.isoformat() if snapshot.created_at else None,
|
||
"created_by": snapshot.created_by,
|
||
}
|
||
|
||
|
||
class CompareSnapshotsRequest(BaseModel):
|
||
snapshot_id_1: str = Field(min_length=1)
|
||
snapshot_id_2: str = Field(min_length=1)
|
||
|
||
|
||
@router.post("/{eval_id}/config-snapshots/compare")
|
||
async def compare_config_snapshots(
|
||
eval_id: str,
|
||
request: CompareSnapshotsRequest,
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
"""Compare two config snapshots and return differences."""
|
||
from agenteval.intelligent_eval import config_snapshot
|
||
from agenteval.storage.db import IntelligentEvalDB
|
||
|
||
# Verify eval exists
|
||
eval_db = session.get(IntelligentEvalDB, eval_id)
|
||
if eval_db is None:
|
||
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
|
||
|
||
# Get both snapshots
|
||
snapshot1 = config_snapshot.get_snapshot(request.snapshot_id_1, session)
|
||
snapshot2 = config_snapshot.get_snapshot(request.snapshot_id_2, session)
|
||
|
||
if snapshot1 is None or snapshot1.eval_id != eval_id:
|
||
raise HTTPException(status_code=404, detail=f"snapshot {request.snapshot_id_1} not found")
|
||
if snapshot2 is None or snapshot2.eval_id != eval_id:
|
||
raise HTTPException(status_code=404, detail=f"snapshot {request.snapshot_id_2} not found")
|
||
|
||
# Compare snapshots
|
||
diffs = config_snapshot.compare_snapshots(snapshot1, snapshot2)
|
||
|
||
return {
|
||
"snapshot_1": {
|
||
"id": snapshot1.id,
|
||
"snapshot_type": snapshot1.snapshot_type,
|
||
"created_at": snapshot1.created_at.isoformat() if snapshot1.created_at else None,
|
||
},
|
||
"snapshot_2": {
|
||
"id": snapshot2.id,
|
||
"snapshot_type": snapshot2.snapshot_type,
|
||
"created_at": snapshot2.created_at.isoformat() if snapshot2.created_at else None,
|
||
},
|
||
"differences": diffs,
|
||
}
|