8e65e2e7b0
All checks were successful
CI / test (push) Successful in 3m47s
openclaw agent has no cron state; a bare 'agenteval-intelligent-worker' message made the worker skill decide then 'wait for the next tick', deadlocking (task assigned, session never created). The trigger message now demands '立即完成当前任务，不要等待下一节拍' and, when all sessions are done, delegates to agenteval-intelligent-analyst. Verified end-to-end on t480: 1h-window eval went executing -> session (2 real turns) -> close -> report -> completed, fully agent-driven, no external IM channel.
268 lines
10 KiB
Python
268 lines
10 KiB
Python
"""FastAPI web backend for AgentEvalTool."""
|
||
|
||
import asyncio
|
||
import logging
|
||
from contextlib import asynccontextmanager
|
||
from pathlib import Path
|
||
|
||
from fastapi import Depends, FastAPI, WebSocket, WebSocketDisconnect
|
||
from fastapi.middleware.cors import CORSMiddleware
|
||
from fastapi.responses import FileResponse, JSONResponse
|
||
|
||
from agenteval.config import get_settings
|
||
from agenteval.storage.db import get_session, init_db
|
||
from agenteval.version import get_build_info, get_version
|
||
from agenteval.web.deps import require_api_key
|
||
from agenteval.web.routers import (
|
||
auth,
|
||
campaigns,
|
||
exploration,
|
||
files,
|
||
intelligent_evals,
|
||
model_configs,
|
||
openclaw_cron_pool,
|
||
proxy,
|
||
reports,
|
||
runs,
|
||
scenarios,
|
||
stats,
|
||
targets,
|
||
)
|
||
from agenteval.web.websocket import ws_manager
|
||
|
||
|
||
def _has_pending_task() -> bool:
|
||
"""True if the intelligent-eval task queue has a pending worker task."""
|
||
from agenteval.intelligent_eval.task_queue import get_next_task
|
||
|
||
session = get_session()
|
||
try:
|
||
return get_next_task(session) is not None
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
async def _trigger_intelligent_worker() -> bool:
|
||
"""Trigger OpenClaw's headless agent to run the intelligent-eval worker skill.
|
||
|
||
方案③（避免外部 IM channel）：平台只当"触发闹钟"——有 pending 任务时，
|
||
用 `docker exec openclaw-eval openclaw agent --agent main -m
|
||
"agenteval-intelligent-worker"` 唤醒 agent，worker skill 由 OpenClaw agent
|
||
自主执行（决策/建会话/真实对话/上报，逻辑零改动）。`--deliver` 默认 false
|
||
因此不经 cron delivery channel。
|
||
|
||
Returns:
|
||
True 若确实触发了 agent（存在 pending 任务）。
|
||
"""
|
||
if not _has_pending_task():
|
||
return False
|
||
|
||
import subprocess
|
||
|
||
_logger = logging.getLogger("agenteval")
|
||
# 触发指令必须带"立即执行"语义：`openclaw agent` 无 cron state，若只发
|
||
# skill 名，agent 会按 worker skill 的"跨节拍"设计决策后等下一拍而死锁。
|
||
# 明确要求"立即完成当前任务"后，agent 会在本次触发内完成会话/分析。
|
||
worker_msg = (
|
||
"执行 agenteval-intelligent-worker skill，立即完成当前任务，不要等待下一节拍："
|
||
"若需执行会话则立即创建会话并开始对话；若所有会话已完成则调用 "
|
||
"agenteval-intelligent-analyst skill 完成分析并生成报告。"
|
||
)
|
||
try:
|
||
proc = await asyncio.to_thread(
|
||
subprocess.run,
|
||
[
|
||
"docker", "exec", "openclaw-eval", "openclaw", "agent",
|
||
"--agent", "main", "-m", worker_msg, "--json",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=300,
|
||
)
|
||
_logger.info("Worker 触发完成 exit=%s", proc.returncode)
|
||
if proc.returncode != 0:
|
||
_logger.warning("Worker 触发 stderr: %s", proc.stderr[-300:])
|
||
except Exception as exc:
|
||
_logger.warning("Worker 触发失败（忽略）: %s", exc)
|
||
return True
|
||
|
||
|
||
async def _intelligent_eval_scan_loop() -> None:
|
||
"""Scan executing intelligent evals, enqueue tasks, and trigger the worker.
|
||
|
||
v1.1.0 缺陷修复：`scan_and_enqueue_tasks` 此前没有调度点，OpenClaw Worker
|
||
每分钟唤醒却永远取不到任务。平台启动后每 60s：① 扫描 executing 的评估入队；
|
||
② 若有 pending 任务则触发 OpenClaw agent 执行 worker skill（方案③，免外部
|
||
channel）。失败不阻断（下次循环继续）。
|
||
"""
|
||
_logger = logging.getLogger("agenteval")
|
||
while True:
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
from agenteval.intelligent_eval.task_queue import scan_and_enqueue_tasks
|
||
|
||
n = scan_and_enqueue_tasks(session)
|
||
if n:
|
||
_logger.info("智能评估扫描：入队 %d 个 Worker 任务", n)
|
||
finally:
|
||
session.close()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("智能评估扫描失败（忽略）: %s", exc)
|
||
try:
|
||
await _trigger_intelligent_worker()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("Worker 触发失败（忽略）: %s", exc)
|
||
await asyncio.sleep(60)
|
||
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(_: FastAPI):
|
||
init_db()
|
||
scan_task = asyncio.create_task(_intelligent_eval_scan_loop())
|
||
# 恢复耐久 Campaign/智能作业，并清理无法安全重放的中断运行（尽力而为，不阻断启动）。
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
from agenteval.evaluation.campaign_runner import campaign_runtime
|
||
from agenteval.evaluation.intelligence_jobs import recover_campaign_intelligence_jobs
|
||
|
||
interrupted_jobs, resumed_jobs = recover_campaign_intelligence_jobs(session)
|
||
recovery = campaign_runtime.recover()
|
||
if recovery.interrupted_runs:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动清理：%d 个中断的运行已标记为 failed", recovery.interrupted_runs
|
||
)
|
||
if interrupted_jobs:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动清理：%d 条中断的分析/对比已标记为 failed",
|
||
interrupted_jobs,
|
||
)
|
||
if recovery.resumed_campaigns:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动恢复：%d 个进行中的评估活动已续跑", recovery.resumed_campaigns
|
||
)
|
||
if resumed_jobs:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动恢复：%d 条排队中的活动分析/对比已续跑",
|
||
resumed_jobs,
|
||
)
|
||
finally:
|
||
session.close()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("启动清理失败（忽略）: %s", exc)
|
||
yield
|
||
# 优雅停止后台扫描，再停其他进程内任务。
|
||
scan_task.cancel()
|
||
try:
|
||
await scan_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
# 优雅停止所有进程内任务：先停活动调度循环，再停在跑的评测运行，
|
||
# 最后停活动智能作业（分析 / 周期对比）和 judge 复核。
|
||
try:
|
||
from agenteval.evaluation.campaign_runner import campaign_runtime
|
||
from agenteval.evaluation.intelligence_jobs import shutdown_campaign_intelligence_jobs
|
||
from agenteval.exploration.judge import judge_registry
|
||
from agenteval.web.routers.runs import run_registry
|
||
|
||
await campaign_runtime.shutdown()
|
||
await run_registry.shutdown_all()
|
||
await shutdown_campaign_intelligence_jobs()
|
||
await judge_registry.shutdown_all()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("活动调度停止失败（忽略）: %s", exc)
|
||
|
||
|
||
app = FastAPI(
|
||
title="AgentEvalTool",
|
||
description="智能体质量评估工具集平台 Web API",
|
||
version=get_version(),
|
||
lifespan=lifespan,
|
||
)
|
||
|
||
settings = get_settings()
|
||
|
||
app.add_middleware(
|
||
CORSMiddleware,
|
||
allow_origins=settings.allowed_origins,
|
||
allow_credentials=True,
|
||
allow_methods=["*"],
|
||
allow_headers=["*"],
|
||
)
|
||
|
||
_api_deps = [Depends(require_api_key)]
|
||
|
||
# Login endpoints must stay open — they are how the client obtains credentials.
|
||
app.include_router(auth.router, prefix="/api/auth", tags=["auth"])
|
||
app.include_router(targets.router, prefix="/api/targets", tags=["targets"], dependencies=_api_deps)
|
||
app.include_router(scenarios.router, prefix="/api/scenarios", tags=["scenarios"], dependencies=_api_deps)
|
||
app.include_router(runs.router, prefix="/api/runs", tags=["runs"], dependencies=_api_deps)
|
||
app.include_router(campaigns.router, prefix="/api/campaigns", tags=["campaigns"], dependencies=_api_deps)
|
||
app.include_router(exploration.router, prefix="/api/exploration", tags=["exploration"], dependencies=_api_deps)
|
||
app.include_router(
|
||
intelligent_evals.router, prefix="/api/intelligent-evals", tags=["intelligent-evals"], dependencies=_api_deps
|
||
)
|
||
app.include_router(
|
||
openclaw_cron_pool.router, prefix="/api/openclaw", tags=["openclaw-cron-pool"], dependencies=_api_deps
|
||
)
|
||
app.include_router(reports.router, prefix="/api/reports", tags=["reports"], dependencies=_api_deps)
|
||
app.include_router(stats.router, prefix="/api/stats", tags=["stats"], dependencies=_api_deps)
|
||
app.include_router(files.router, prefix="/api/files", tags=["files"], dependencies=_api_deps)
|
||
app.include_router(
|
||
model_configs.router,
|
||
prefix="/api/model-configs",
|
||
tags=["model-configs"],
|
||
dependencies=_api_deps,
|
||
)
|
||
|
||
|
||
@app.websocket("/openclaw")
|
||
async def openclaw_ws_root(websocket: WebSocket):
|
||
url = proxy.get_ws_upstream()
|
||
if websocket.query_params:
|
||
url += f"?{websocket.query_params}"
|
||
await proxy.ws_bridge(websocket, url)
|
||
|
||
|
||
@app.websocket("/openclaw/{path:path}")
|
||
async def openclaw_ws_proxy(websocket: WebSocket, path: str):
|
||
url = f"{proxy.get_ws_upstream()}/{path}"
|
||
if websocket.query_params:
|
||
url += f"?{websocket.query_params}"
|
||
await proxy.ws_bridge(websocket, url)
|
||
|
||
|
||
app.include_router(proxy.router, prefix="/openclaw", tags=["proxy"])
|
||
|
||
_default_dist = Path(__file__).resolve().parent.parent.parent.parent / "frontend" / "web" / "dist"
|
||
WEB_DIST = Path(settings.frontend_dist_path) if settings.frontend_dist_path else _default_dist
|
||
if WEB_DIST.exists():
|
||
from fastapi.staticfiles import StaticFiles
|
||
|
||
app.mount("/assets", StaticFiles(directory=WEB_DIST / "assets"), name="assets")
|
||
|
||
|
||
@app.get("/api/health", tags=["health"])
|
||
def health() -> dict:
|
||
return {"status": "ok", **get_build_info()}
|
||
|
||
|
||
@app.websocket("/ws/runs/{run_id}")
|
||
async def websocket_run_progress(websocket: WebSocket, run_id: str):
|
||
await ws_manager.connect(run_id, websocket)
|
||
try:
|
||
while True:
|
||
await websocket.receive_text()
|
||
except WebSocketDisconnect:
|
||
ws_manager.disconnect(run_id, websocket)
|
||
|
||
|
||
@app.get("/{full_path:path}")
|
||
def serve_spa(full_path: str):
|
||
"""Serve the React SPA for all non-API routes."""
|
||
index_file = WEB_DIST / "index.html"
|
||
if index_file.exists():
|
||
return FileResponse(index_file)
|
||
return JSONResponse({"detail": "frontend not built"}, status_code=404)
|