333 lines
13 KiB
Python
333 lines
13 KiB
Python
"""FastAPI web backend for AgentEvalTool."""
|
||
|
||
import asyncio
|
||
import logging
|
||
from contextlib import asynccontextmanager
|
||
from pathlib import Path
|
||
|
||
from fastapi import Depends, FastAPI, WebSocket, WebSocketDisconnect
|
||
from fastapi.middleware.cors import CORSMiddleware
|
||
from fastapi.responses import FileResponse, JSONResponse
|
||
|
||
from agenteval.config import get_settings
|
||
from agenteval.storage.db import get_session, init_db
|
||
from agenteval.version import get_build_info, get_version
|
||
from agenteval.web.deps import require_api_key
|
||
from agenteval.web.routers import (
|
||
auth,
|
||
campaigns,
|
||
exploration,
|
||
files,
|
||
intelligent_evals,
|
||
model_configs,
|
||
openclaw_cron_pool,
|
||
proxy,
|
||
reports,
|
||
runs,
|
||
scenarios,
|
||
stats,
|
||
targets,
|
||
)
|
||
from agenteval.web.websocket import ws_manager
|
||
|
||
|
||
def _has_pending_task() -> bool:
|
||
"""True if the intelligent-eval task queue has a pending worker task."""
|
||
from agenteval.intelligent_eval.task_queue import get_next_task
|
||
|
||
session = get_session()
|
||
try:
|
||
return get_next_task(session) is not None
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
def _supplement_decision_logs(session) -> int:
|
||
"""Platform audit backfill for decision logs.
|
||
|
||
方案③的决策日志由 OpenClaw agent 上报（LLM 自主，尽力而为）——异常路径
|
||
（如卡死恢复后重试）agent 可能跳过上报，导致决策过程页面为空。这里按评估
|
||
状态推导决策并补录：欠账时补 execute_session，所有会话完成后补
|
||
start_analysis。只补"该类型缺失"的，不重复；且只记录状态，不改变 agent
|
||
的实际执行。
|
||
|
||
Returns:
|
||
补录的决策日志条数。
|
||
"""
|
||
from sqlmodel import select
|
||
|
||
from agenteval.intelligent_eval.decision_logs import create_decision_log
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalDecisionLogDB, IntelligentEvalSessionDB
|
||
|
||
evals = session.exec(
|
||
select(IntelligentEvalDB).where(IntelligentEvalDB.status == IntelligentEvalStatus.EXECUTING.value)
|
||
).all()
|
||
|
||
added = 0
|
||
for ev in evals:
|
||
plan = ev.get_plan() if ev.plan else {}
|
||
estimated = plan.get("estimated_sessions", 0)
|
||
sessions = session.exec(
|
||
select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.eval_id == ev.id)
|
||
).all()
|
||
completed = sum(1 for s in sessions if s.status == "completed")
|
||
types = {
|
||
l.decision_type
|
||
for l in session.exec(
|
||
select(IntelligentEvalDecisionLogDB).where(
|
||
IntelligentEvalDecisionLogDB.eval_id == ev.id
|
||
)
|
||
).all()
|
||
}
|
||
if "execute_session" not in types and completed < estimated:
|
||
create_decision_log(
|
||
ev.id, "execute_session", "平台兜底：时段欠账需执行会话", "platform",
|
||
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
|
||
session,
|
||
)
|
||
added += 1
|
||
elif "start_analysis" not in types and sessions and completed >= estimated:
|
||
create_decision_log(
|
||
ev.id, "start_analysis", "平台兜底：所有会话已完成开始分析", "platform",
|
||
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
|
||
session,
|
||
)
|
||
added += 1
|
||
return added
|
||
|
||
|
||
async def _trigger_intelligent_worker() -> bool:
|
||
"""Trigger OpenClaw's headless agent to run the intelligent-eval worker skill.
|
||
|
||
方案③（避免外部 IM channel）：平台只当"触发闹钟"——有 pending 任务时，
|
||
用 `docker exec openclaw-eval openclaw agent --agent main -m
|
||
"agenteval-intelligent-worker"` 唤醒 agent，worker skill 由 OpenClaw agent
|
||
自主执行（决策/建会话/真实对话/上报，逻辑零改动）。`--deliver` 默认 false
|
||
因此不经 cron delivery channel。
|
||
|
||
Returns:
|
||
True 若确实触发了 agent（存在 pending 任务）。
|
||
"""
|
||
if not _has_pending_task():
|
||
return False
|
||
|
||
import subprocess
|
||
|
||
_logger = logging.getLogger("agenteval")
|
||
# 触发指令必须带"立即执行"语义：`openclaw agent` 无 cron state，若只发
|
||
# skill 名，agent 会按 worker skill 的"跨节拍"设计决策后等下一拍而死锁。
|
||
# 明确要求"立即完成当前任务"后，agent 会在本次触发内完成会话/分析。
|
||
worker_msg = (
|
||
"执行 agenteval-intelligent-worker skill，立即完成当前任务，不要等待下一节拍："
|
||
"若需执行会话则立即创建会话并开始对话；若所有会话已完成则调用 "
|
||
"agenteval-intelligent-analyst skill 完成分析并生成报告。"
|
||
)
|
||
try:
|
||
proc = await asyncio.to_thread(
|
||
subprocess.run,
|
||
[
|
||
"docker", "exec", "openclaw-eval", "openclaw", "agent",
|
||
"--agent", "main", "-m", worker_msg, "--json",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=300,
|
||
)
|
||
_logger.info("Worker 触发完成 exit=%s", proc.returncode)
|
||
if proc.returncode != 0:
|
||
_logger.warning("Worker 触发 stderr: %s", proc.stderr[-300:])
|
||
except Exception as exc:
|
||
_logger.warning("Worker 触发失败（忽略）: %s", exc)
|
||
return True
|
||
|
||
|
||
async def _intelligent_eval_scan_loop() -> None:
|
||
"""Scan executing intelligent evals, enqueue tasks, and trigger the worker.
|
||
|
||
v1.1.0 缺陷修复：`scan_and_enqueue_tasks` 此前没有调度点，OpenClaw Worker
|
||
每分钟唤醒却永远取不到任务。平台启动后每 60s：① 扫描 executing 的评估入队；
|
||
② 若有 pending 任务则触发 OpenClaw agent 执行 worker skill（方案③，免外部
|
||
channel）。失败不阻断（下次循环继续）。
|
||
"""
|
||
_logger = logging.getLogger("agenteval")
|
||
while True:
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
from agenteval.intelligent_eval.task_queue import (
|
||
requeue_stale_assigned_tasks,
|
||
scan_and_enqueue_tasks,
|
||
)
|
||
|
||
r = requeue_stale_assigned_tasks(session)
|
||
if r:
|
||
_logger.info("卡死恢复：%d 个 assigned 任务重新入队", r)
|
||
n = scan_and_enqueue_tasks(session)
|
||
if n:
|
||
_logger.info("智能评估扫描：入队 %d 个 Worker 任务", n)
|
||
# 审计兜底：agent 未上报决策日志时，平台按评估状态补录
|
||
added = _supplement_decision_logs(session)
|
||
if added:
|
||
_logger.info("决策日志兜底：补录 %d 条", added)
|
||
finally:
|
||
session.close()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("智能评估扫描失败（忽略）: %s", exc)
|
||
try:
|
||
await _trigger_intelligent_worker()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("Worker 触发失败（忽略）: %s", exc)
|
||
await asyncio.sleep(60)
|
||
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(_: FastAPI):
|
||
init_db()
|
||
scan_task = asyncio.create_task(_intelligent_eval_scan_loop())
|
||
# 恢复耐久 Campaign/智能作业，并清理无法安全重放的中断运行（尽力而为，不阻断启动）。
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
from agenteval.evaluation.campaign_runner import campaign_runtime
|
||
from agenteval.evaluation.intelligence_jobs import recover_campaign_intelligence_jobs
|
||
|
||
interrupted_jobs, resumed_jobs = recover_campaign_intelligence_jobs(session)
|
||
recovery = campaign_runtime.recover()
|
||
if recovery.interrupted_runs:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动清理：%d 个中断的运行已标记为 failed", recovery.interrupted_runs
|
||
)
|
||
if interrupted_jobs:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动清理：%d 条中断的分析/对比已标记为 failed",
|
||
interrupted_jobs,
|
||
)
|
||
if recovery.resumed_campaigns:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动恢复：%d 个进行中的评估活动已续跑", recovery.resumed_campaigns
|
||
)
|
||
if resumed_jobs:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动恢复：%d 条排队中的活动分析/对比已续跑",
|
||
resumed_jobs,
|
||
)
|
||
finally:
|
||
session.close()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("启动清理失败（忽略）: %s", exc)
|
||
yield
|
||
# 优雅停止后台扫描，再停其他进程内任务。
|
||
scan_task.cancel()
|
||
try:
|
||
await scan_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
# 优雅停止所有进程内任务：先停活动调度循环，再停在跑的评测运行，
|
||
# 最后停活动智能作业（分析 / 周期对比）和 judge 复核。
|
||
try:
|
||
from agenteval.evaluation.campaign_runner import campaign_runtime
|
||
from agenteval.evaluation.intelligence_jobs import shutdown_campaign_intelligence_jobs
|
||
from agenteval.exploration.judge import judge_registry
|
||
from agenteval.web.routers.runs import run_registry
|
||
|
||
await campaign_runtime.shutdown()
|
||
await run_registry.shutdown_all()
|
||
await shutdown_campaign_intelligence_jobs()
|
||
await judge_registry.shutdown_all()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("活动调度停止失败（忽略）: %s", exc)
|
||
|
||
|
||
app = FastAPI(
|
||
title="AgentEvalTool",
|
||
description="智能体质量评估工具集平台 Web API",
|
||
version=get_version(),
|
||
lifespan=lifespan,
|
||
)
|
||
|
||
settings = get_settings()
|
||
|
||
app.add_middleware(
|
||
CORSMiddleware,
|
||
allow_origins=settings.allowed_origins,
|
||
allow_credentials=True,
|
||
allow_methods=["*"],
|
||
allow_headers=["*"],
|
||
)
|
||
|
||
_api_deps = [Depends(require_api_key)]
|
||
|
||
# Login endpoints must stay open — they are how the client obtains credentials.
|
||
app.include_router(auth.router, prefix="/api/auth", tags=["auth"])
|
||
app.include_router(targets.router, prefix="/api/targets", tags=["targets"], dependencies=_api_deps)
|
||
app.include_router(scenarios.router, prefix="/api/scenarios", tags=["scenarios"], dependencies=_api_deps)
|
||
app.include_router(runs.router, prefix="/api/runs", tags=["runs"], dependencies=_api_deps)
|
||
app.include_router(campaigns.router, prefix="/api/campaigns", tags=["campaigns"], dependencies=_api_deps)
|
||
app.include_router(exploration.router, prefix="/api/exploration", tags=["exploration"], dependencies=_api_deps)
|
||
app.include_router(
|
||
intelligent_evals.router, prefix="/api/intelligent-evals", tags=["intelligent-evals"], dependencies=_api_deps
|
||
)
|
||
app.include_router(
|
||
openclaw_cron_pool.router, prefix="/api/openclaw", tags=["openclaw-cron-pool"], dependencies=_api_deps
|
||
)
|
||
app.include_router(reports.router, prefix="/api/reports", tags=["reports"], dependencies=_api_deps)
|
||
app.include_router(stats.router, prefix="/api/stats", tags=["stats"], dependencies=_api_deps)
|
||
app.include_router(files.router, prefix="/api/files", tags=["files"], dependencies=_api_deps)
|
||
app.include_router(
|
||
model_configs.router,
|
||
prefix="/api/model-configs",
|
||
tags=["model-configs"],
|
||
dependencies=_api_deps,
|
||
)
|
||
|
||
|
||
@app.websocket("/openclaw")
|
||
async def openclaw_ws_root(websocket: WebSocket):
|
||
url = proxy.get_ws_upstream()
|
||
if websocket.query_params:
|
||
url += f"?{websocket.query_params}"
|
||
await proxy.ws_bridge(websocket, url)
|
||
|
||
|
||
@app.websocket("/openclaw/{path:path}")
|
||
async def openclaw_ws_proxy(websocket: WebSocket, path: str):
|
||
url = f"{proxy.get_ws_upstream()}/{path}"
|
||
if websocket.query_params:
|
||
url += f"?{websocket.query_params}"
|
||
await proxy.ws_bridge(websocket, url)
|
||
|
||
|
||
app.include_router(proxy.router, prefix="/openclaw", tags=["proxy"])
|
||
|
||
_default_dist = Path(__file__).resolve().parent.parent.parent.parent / "frontend" / "web" / "dist"
|
||
WEB_DIST = Path(settings.frontend_dist_path) if settings.frontend_dist_path else _default_dist
|
||
if WEB_DIST.exists():
|
||
from fastapi.staticfiles import StaticFiles
|
||
|
||
app.mount("/assets", StaticFiles(directory=WEB_DIST / "assets"), name="assets")
|
||
|
||
|
||
@app.get("/api/health", tags=["health"])
|
||
def health() -> dict:
|
||
return {"status": "ok", **get_build_info()}
|
||
|
||
|
||
@app.websocket("/ws/runs/{run_id}")
|
||
async def websocket_run_progress(websocket: WebSocket, run_id: str):
|
||
await ws_manager.connect(run_id, websocket)
|
||
try:
|
||
while True:
|
||
await websocket.receive_text()
|
||
except WebSocketDisconnect:
|
||
ws_manager.disconnect(run_id, websocket)
|
||
|
||
|
||
@app.get("/{full_path:path}")
|
||
def serve_spa(full_path: str):
|
||
"""Serve the React SPA for all non-API routes."""
|
||
index_file = WEB_DIST / "index.html"
|
||
if index_file.exists():
|
||
return FileResponse(index_file)
|
||
return JSONResponse({"detail": "frontend not built"}, status_code=404)
|