5de46d514a
All checks were successful
CI / test (push) Successful in 4m0s
t480 排查：评估 cd636d71 一直'等待 OpenClaw 创建会话'，scan loop 每 60s 触发 worker，但 worker 被触发后 0 次工具调用、直接幻觉输出'评估 pending_approval' （实际 executing），任务永不认领。 根因：openclaw agent --agent main 复用 main 持久 session，多次触发累积上下文 缓存（~12 万 token）后 LLM 不再执行 worker skill 的 API 步骤。 验证：独立 --session-id 触发 worker → 正常取任务、建会话、close。 修复：worker/planner 触发命令加 --session-id（每次唯一 agenteval-worker-*/planner-*）， 避免 main session 污染；timeout 300→600（独立 session 首次加载 skill 更慢）。 测试：触发断言含 --session-id，901 passed
490 lines
20 KiB
Python
490 lines
20 KiB
Python
"""FastAPI web backend for AgentEvalTool."""
|
||
|
||
import asyncio
|
||
import logging
|
||
import time
|
||
from contextlib import asynccontextmanager
|
||
from pathlib import Path
|
||
|
||
from fastapi import Depends, FastAPI, WebSocket, WebSocketDisconnect
|
||
from fastapi.middleware.cors import CORSMiddleware
|
||
from fastapi.responses import FileResponse, JSONResponse
|
||
|
||
from agenteval.config import get_settings
|
||
from agenteval.storage.db import get_session, init_db
|
||
from agenteval.version import get_build_info, get_version
|
||
from agenteval.web.deps import require_api_key
|
||
from agenteval.web.routers import (
|
||
auth,
|
||
campaigns,
|
||
exploration,
|
||
files,
|
||
intelligent_evals,
|
||
model_configs,
|
||
openclaw_cron_pool,
|
||
proxy,
|
||
reports,
|
||
runs,
|
||
scenarios,
|
||
stats,
|
||
targets,
|
||
)
|
||
from agenteval.web.websocket import ws_manager
|
||
|
||
|
||
def _has_pending_task() -> bool:
|
||
"""True if the intelligent-eval task queue has a pending worker task."""
|
||
from agenteval.intelligent_eval.task_queue import get_next_task
|
||
|
||
session = get_session()
|
||
try:
|
||
return get_next_task(session) is not None
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
def _has_planning_eval() -> bool:
|
||
"""True if any intelligent eval is waiting in ``planning`` (needs the OpenClaw planner)."""
|
||
from sqlmodel import select
|
||
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB
|
||
|
||
session = get_session()
|
||
try:
|
||
ev = session.exec(
|
||
select(IntelligentEvalDB).where(
|
||
IntelligentEvalDB.status == IntelligentEvalStatus.PLANNING.value
|
||
)
|
||
).first()
|
||
return ev is not None
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
def _supplement_decision_logs(session) -> int:
|
||
"""Platform audit backfill for decision logs.
|
||
|
||
方案③的决策日志由 OpenClaw agent 上报（LLM 自主，尽力而为）——异常路径
|
||
（如卡死恢复后重试）agent 可能跳过上报，导致决策过程页面为空。这里按评估
|
||
状态推导决策并补录：
|
||
|
||
- EXECUTING：欠账（completed < estimated）补 execute_session，所有会话
|
||
完成后补 start_analysis。
|
||
- COMPLETED：历史评估/异常路径可能完全没有决策日志，回填 execute_session
|
||
（按 plan 时段逐条）+ start_analysis，让旧报告也有决策过程可看。
|
||
|
||
只补"该类型缺失"的，不重复；且只记录状态，不改变 agent 的实际执行。
|
||
|
||
Returns:
|
||
补录的决策日志条数。
|
||
"""
|
||
from sqlmodel import select
|
||
|
||
from agenteval.intelligent_eval.decision_logs import create_decision_log
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalDecisionLogDB, IntelligentEvalSessionDB
|
||
|
||
evals = session.exec(
|
||
select(IntelligentEvalDB).where(
|
||
IntelligentEvalDB.status.in_(
|
||
[
|
||
IntelligentEvalStatus.EXECUTING.value,
|
||
IntelligentEvalStatus.COMPLETED.value,
|
||
]
|
||
)
|
||
)
|
||
).all()
|
||
|
||
added = 0
|
||
for ev in evals:
|
||
plan = ev.get_plan() if ev.plan else {}
|
||
estimated = plan.get("estimated_sessions", 0)
|
||
sessions = session.exec(select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.eval_id == ev.id)).all()
|
||
completed = sum(1 for s in sessions if s.status == "completed")
|
||
types = {
|
||
x.decision_type
|
||
for x in session.exec(
|
||
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
|
||
).all()
|
||
}
|
||
if ev.status == IntelligentEvalStatus.EXECUTING.value:
|
||
if "execute_session" not in types and completed < estimated:
|
||
create_decision_log(
|
||
ev.id,
|
||
"execute_session",
|
||
"平台兜底：时段欠账需执行会话",
|
||
"platform",
|
||
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
|
||
session,
|
||
)
|
||
added += 1
|
||
elif "start_analysis" not in types and sessions and completed >= estimated:
|
||
create_decision_log(
|
||
ev.id,
|
||
"start_analysis",
|
||
"平台兜底：所有会话已完成开始分析",
|
||
"platform",
|
||
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
|
||
session,
|
||
)
|
||
added += 1
|
||
elif ev.status == IntelligentEvalStatus.COMPLETED.value:
|
||
# 历史回填：completed 评估决策日志全缺失时，按时段补 execute_session
|
||
if "execute_session" not in types:
|
||
slots = plan.get("time_distribution") or []
|
||
if slots:
|
||
for slot in slots:
|
||
create_decision_log(
|
||
ev.id,
|
||
"execute_session",
|
||
f"平台兜底：时段{slot.get('time_slot', '')}执行会话（历史回填）",
|
||
"platform",
|
||
{
|
||
"platform_supplemented": True,
|
||
"time_slot": slot.get("time_slot"),
|
||
"sessions": slot.get("sessions"),
|
||
"completed": completed,
|
||
"estimated": estimated,
|
||
},
|
||
session,
|
||
)
|
||
added += 1
|
||
else:
|
||
create_decision_log(
|
||
ev.id,
|
||
"execute_session",
|
||
"平台兜底：执行会话（历史回填）",
|
||
"platform",
|
||
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
|
||
session,
|
||
)
|
||
added += 1
|
||
if "start_analysis" not in types and sessions:
|
||
create_decision_log(
|
||
ev.id,
|
||
"start_analysis",
|
||
"平台兜底：所有会话已完成开始分析（历史回填）",
|
||
"platform",
|
||
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
|
||
session,
|
||
)
|
||
added += 1
|
||
return added
|
||
|
||
|
||
async def _trigger_intelligent_worker() -> bool:
|
||
"""Trigger OpenClaw's headless agent to run the intelligent-eval worker skill.
|
||
|
||
方案③（避免外部 IM channel）：平台只当"触发闹钟"——有 pending 任务时，
|
||
用 `docker exec openclaw-eval openclaw agent --agent main -m
|
||
"agenteval-intelligent-worker"` 唤醒 agent，worker skill 由 OpenClaw agent
|
||
自主执行（决策/建会话/真实对话/上报，逻辑零改动）。`--deliver` 默认 false
|
||
因此不经 cron delivery channel。
|
||
|
||
Returns:
|
||
True 若确实触发了 agent（存在 pending 任务）。
|
||
"""
|
||
if not _has_pending_task():
|
||
return False
|
||
|
||
import subprocess
|
||
|
||
_logger = logging.getLogger("agenteval")
|
||
# 触发指令必须带"立即执行"语义：`openclaw agent` 无 cron state，若只发
|
||
# skill 名，agent 会按 worker skill 的"跨节拍"设计决策后等下一拍而死锁。
|
||
# 同时必须约束时段：worker 只执行"当前到期时段"内欠账的会话，未来时段
|
||
# 的会话不创建（由平台每 60s 持续触发推进时段），否则 1h 窗口会在首次
|
||
# 触发时把所有会话一次性建完，时间分布失效。
|
||
worker_msg = (
|
||
"执行 agenteval-intelligent-worker skill，立即完成当前任务，不要等待下一节拍："
|
||
"仅执行当前时间对应时段（time_distribution 中当前 offset 所在时段）内欠账的会话"
|
||
"——按评估 started_at 与当前时间精确判断当前时段，只创建该时段计划内的会话，"
|
||
"绝不创建未来时段的会话，未来时段到期后平台会再次触发你；"
|
||
"若所有会话已完成则调用 agenteval-intelligent-analyst skill 完成分析并生成报告。"
|
||
)
|
||
try:
|
||
proc = await asyncio.to_thread(
|
||
subprocess.run,
|
||
[
|
||
"docker",
|
||
"exec",
|
||
"openclaw-eval",
|
||
"openclaw",
|
||
"agent",
|
||
"--agent",
|
||
"main",
|
||
# 必须用独立 session：`--agent main` 复用 main 持久 session，多次触发
|
||
# 累积上下文缓存（~12 万 token）后 LLM 不再执行 worker skill 的 API
|
||
# 步骤，直接幻觉输出（如"评估 pending_approval"）而不取任务建会话。
|
||
"--session-id",
|
||
f"agenteval-worker-{int(time.time())}",
|
||
"-m",
|
||
worker_msg,
|
||
"--json",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=600,
|
||
)
|
||
_logger.info("Worker 触发完成 exit=%s", proc.returncode)
|
||
if proc.returncode != 0:
|
||
_logger.warning("Worker 触发 stderr: %s", proc.stderr[-300:])
|
||
except Exception as exc:
|
||
_logger.warning("Worker 触发失败（忽略）: %s", exc)
|
||
return True
|
||
|
||
|
||
async def _trigger_intelligent_planner() -> bool:
|
||
"""Trigger OpenClaw's headless agent to run the planner skill.
|
||
|
||
方案③只自动化了 executing（worker）→ completed（analyst），**缺少 planning
|
||
阶段的 planner 触发**——新建或被打回的评估会永远停在 planning。这里对
|
||
planning 状态评估触发 `agenteval-intelligent-planner` skill：planner 自会
|
||
拉取 planning 评估列表、读取四件套、产出粗计划并 PUT /plan 提交（planner
|
||
skill 定义见 OpenClaw workspace skills）。处理完评估离开 planning 后不再触发。
|
||
|
||
Returns:
|
||
True 若确实触发了 agent（存在 planning 评估）。
|
||
"""
|
||
if not _has_planning_eval():
|
||
return False
|
||
|
||
import subprocess
|
||
|
||
_logger = logging.getLogger("agenteval")
|
||
# 同 worker：`openclaw agent` 无 cron state，须带"立即完成"语义，否则 planner
|
||
# 会"决策后等下一拍"而死锁。
|
||
planner_msg = (
|
||
"执行 agenteval-intelligent-planner skill，立即完成当前任务，不要等待下一节拍："
|
||
"为 planning 状态的智能评估读取输入、产出粗计划并提交平台审批。"
|
||
)
|
||
try:
|
||
proc = await asyncio.to_thread(
|
||
subprocess.run,
|
||
[
|
||
"docker",
|
||
"exec",
|
||
"openclaw-eval",
|
||
"openclaw",
|
||
"agent",
|
||
"--agent",
|
||
"main",
|
||
# 同 worker：独立 session 避免 main 持久 session 上下文缓存污染
|
||
"--session-id",
|
||
f"agenteval-planner-{int(time.time())}",
|
||
"-m",
|
||
planner_msg,
|
||
"--json",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=600,
|
||
)
|
||
_logger.info("Planner 触发完成 exit=%s", proc.returncode)
|
||
if proc.returncode != 0:
|
||
_logger.warning("Planner 触发 stderr: %s", proc.stderr[-300:])
|
||
except Exception as exc:
|
||
_logger.warning("Planner 触发失败（忽略）: %s", exc)
|
||
return True
|
||
|
||
|
||
async def _intelligent_eval_scan_loop() -> None:
|
||
"""Scan intelligent evals, enqueue tasks, and trigger planner/worker.
|
||
|
||
v1.1.0 缺陷修复：`scan_and_enqueue_tasks` 此前没有调度点，OpenClaw Worker
|
||
每分钟唤醒却永远取不到任务。平台启动后每 60s：
|
||
① 有 planning 评估则触发 OpenClaw planner skill 产出粗计划（planning→待审批）；
|
||
② 扫描 executing 的评估入队；③ 若有 pending 任务则触发 worker skill 执行会话/
|
||
分析（方案③，免外部 channel）。失败不阻断（下次循环继续）。
|
||
"""
|
||
_logger = logging.getLogger("agenteval")
|
||
while True:
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
from agenteval.intelligent_eval.task_queue import (
|
||
requeue_stale_assigned_tasks,
|
||
scan_and_enqueue_tasks,
|
||
settle_tasks_for_finished_evals,
|
||
)
|
||
|
||
r = requeue_stale_assigned_tasks(session)
|
||
if r:
|
||
_logger.info("卡死恢复：%d 个 assigned 任务重新入队", r)
|
||
n = scan_and_enqueue_tasks(session)
|
||
if n:
|
||
_logger.info("智能评估扫描：入队 %d 个 Worker 任务", n)
|
||
# 清理：评估已结束（非 executing）的 pending/assigned 任务回收为 completed
|
||
settled = settle_tasks_for_finished_evals(session)
|
||
if settled:
|
||
_logger.info("任务回收：%d 个已结束评估的任务标记完成", settled)
|
||
# 审计兜底：agent 未上报决策日志时，平台按评估状态补录
|
||
added = _supplement_decision_logs(session)
|
||
if added:
|
||
_logger.info("决策日志兜底：补录 %d 条", added)
|
||
finally:
|
||
session.close()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("智能评估扫描失败（忽略）: %s", exc)
|
||
try:
|
||
await _trigger_intelligent_worker()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("Worker 触发失败（忽略）: %s", exc)
|
||
try:
|
||
await _trigger_intelligent_planner()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("Planner 触发失败（忽略）: %s", exc)
|
||
await asyncio.sleep(60)
|
||
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(_: FastAPI):
|
||
init_db()
|
||
scan_task = asyncio.create_task(_intelligent_eval_scan_loop())
|
||
# 恢复耐久 Campaign/智能作业，并清理无法安全重放的中断运行（尽力而为，不阻断启动）。
|
||
try:
|
||
session = get_session()
|
||
try:
|
||
from agenteval.evaluation.campaign_runner import campaign_runtime
|
||
from agenteval.evaluation.intelligence_jobs import recover_campaign_intelligence_jobs
|
||
|
||
interrupted_jobs, resumed_jobs = recover_campaign_intelligence_jobs(session)
|
||
recovery = campaign_runtime.recover()
|
||
if recovery.interrupted_runs:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动清理：%d 个中断的运行已标记为 failed", recovery.interrupted_runs
|
||
)
|
||
if interrupted_jobs:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动清理：%d 条中断的分析/对比已标记为 failed",
|
||
interrupted_jobs,
|
||
)
|
||
if recovery.resumed_campaigns:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动恢复：%d 个进行中的评估活动已续跑", recovery.resumed_campaigns
|
||
)
|
||
if resumed_jobs:
|
||
logging.getLogger("agenteval").warning(
|
||
"启动恢复：%d 条排队中的活动分析/对比已续跑",
|
||
resumed_jobs,
|
||
)
|
||
finally:
|
||
session.close()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("启动清理失败（忽略）: %s", exc)
|
||
yield
|
||
# 优雅停止后台扫描，再停其他进程内任务。
|
||
scan_task.cancel()
|
||
try:
|
||
await scan_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
# 优雅停止所有进程内任务：先停活动调度循环，再停在跑的评测运行，
|
||
# 最后停活动智能作业（分析 / 周期对比）和 judge 复核。
|
||
try:
|
||
from agenteval.evaluation.campaign_runner import campaign_runtime
|
||
from agenteval.evaluation.intelligence_jobs import shutdown_campaign_intelligence_jobs
|
||
from agenteval.exploration.judge import judge_registry
|
||
from agenteval.web.routers.runs import run_registry
|
||
|
||
await campaign_runtime.shutdown()
|
||
await run_registry.shutdown_all()
|
||
await shutdown_campaign_intelligence_jobs()
|
||
await judge_registry.shutdown_all()
|
||
except Exception as exc:
|
||
logging.getLogger("agenteval").warning("活动调度停止失败（忽略）: %s", exc)
|
||
|
||
|
||
app = FastAPI(
|
||
title="AgentEvalTool",
|
||
description="智能体质量评估工具集平台 Web API",
|
||
version=get_version(),
|
||
lifespan=lifespan,
|
||
)
|
||
|
||
settings = get_settings()
|
||
|
||
app.add_middleware(
|
||
CORSMiddleware,
|
||
allow_origins=settings.allowed_origins,
|
||
allow_credentials=True,
|
||
allow_methods=["*"],
|
||
allow_headers=["*"],
|
||
)
|
||
|
||
_api_deps = [Depends(require_api_key)]
|
||
|
||
# Login endpoints must stay open — they are how the client obtains credentials.
|
||
app.include_router(auth.router, prefix="/api/auth", tags=["auth"])
|
||
app.include_router(targets.router, prefix="/api/targets", tags=["targets"], dependencies=_api_deps)
|
||
app.include_router(scenarios.router, prefix="/api/scenarios", tags=["scenarios"], dependencies=_api_deps)
|
||
app.include_router(runs.router, prefix="/api/runs", tags=["runs"], dependencies=_api_deps)
|
||
app.include_router(campaigns.router, prefix="/api/campaigns", tags=["campaigns"], dependencies=_api_deps)
|
||
app.include_router(exploration.router, prefix="/api/exploration", tags=["exploration"], dependencies=_api_deps)
|
||
app.include_router(
|
||
intelligent_evals.router, prefix="/api/intelligent-evals", tags=["intelligent-evals"], dependencies=_api_deps
|
||
)
|
||
app.include_router(
|
||
openclaw_cron_pool.router, prefix="/api/openclaw", tags=["openclaw-cron-pool"], dependencies=_api_deps
|
||
)
|
||
app.include_router(reports.router, prefix="/api/reports", tags=["reports"], dependencies=_api_deps)
|
||
app.include_router(stats.router, prefix="/api/stats", tags=["stats"], dependencies=_api_deps)
|
||
app.include_router(files.router, prefix="/api/files", tags=["files"], dependencies=_api_deps)
|
||
app.include_router(
|
||
model_configs.router,
|
||
prefix="/api/model-configs",
|
||
tags=["model-configs"],
|
||
dependencies=_api_deps,
|
||
)
|
||
|
||
|
||
@app.websocket("/openclaw")
|
||
async def openclaw_ws_root(websocket: WebSocket):
|
||
url = proxy.get_ws_upstream()
|
||
if websocket.query_params:
|
||
url += f"?{websocket.query_params}"
|
||
await proxy.ws_bridge(websocket, url)
|
||
|
||
|
||
@app.websocket("/openclaw/{path:path}")
|
||
async def openclaw_ws_proxy(websocket: WebSocket, path: str):
|
||
url = f"{proxy.get_ws_upstream()}/{path}"
|
||
if websocket.query_params:
|
||
url += f"?{websocket.query_params}"
|
||
await proxy.ws_bridge(websocket, url)
|
||
|
||
|
||
app.include_router(proxy.router, prefix="/openclaw", tags=["proxy"])
|
||
|
||
_default_dist = Path(__file__).resolve().parent.parent.parent.parent / "frontend" / "web" / "dist"
|
||
WEB_DIST = Path(settings.frontend_dist_path) if settings.frontend_dist_path else _default_dist
|
||
if WEB_DIST.exists():
|
||
from fastapi.staticfiles import StaticFiles
|
||
|
||
app.mount("/assets", StaticFiles(directory=WEB_DIST / "assets"), name="assets")
|
||
|
||
|
||
@app.get("/api/health", tags=["health"])
|
||
def health() -> dict:
|
||
return {"status": "ok", **get_build_info()}
|
||
|
||
|
||
@app.websocket("/ws/runs/{run_id}")
|
||
async def websocket_run_progress(websocket: WebSocket, run_id: str):
|
||
await ws_manager.connect(run_id, websocket)
|
||
try:
|
||
while True:
|
||
await websocket.receive_text()
|
||
except WebSocketDisconnect:
|
||
ws_manager.disconnect(run_id, websocket)
|
||
|
||
|
||
@app.get("/{full_path:path}")
|
||
def serve_spa(full_path: str):
|
||
"""Serve the React SPA for all non-API routes."""
|
||
index_file = WEB_DIST / "index.html"
|
||
if index_file.exists():
|
||
return FileResponse(index_file)
|
||
return JSONResponse({"detail": "frontend not built"}, status_code=404)
|