AgentEvalTool/backend/agenteval/web/app.py
sinohqb eb4944a8bd feat(intelligent-eval): terminal-state discipline watchdogs (ADR-0011)
常见故障自愈有上限,超限收敛终态且可见:任务 attempts 上限、会话过期、
planning 双闸、executing 超窗兜底、触发失败计数判死、孤儿 agent 双管、
fire-and-forget 触发;open_session 预算硬闸门、settle 按终态区分、报告
scores 归一化;cron 池遗留面全删。
2026-08-20 14:34:17 +08:00

585 lines
25 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""FastAPI web backend for AgentEvalTool."""
import asyncio
import logging
import time
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Optional
from fastapi import Depends, FastAPI, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, JSONResponse
from agenteval.config import get_settings
from agenteval.storage.db import get_session, init_db
from agenteval.version import get_build_info, get_version
from agenteval.web.deps import require_api_key
from agenteval.web.routers import (
auth,
campaigns,
exploration,
files,
intelligent_evals,
model_configs,
proxy,
reports,
runs,
scenarios,
stats,
targets,
)
from agenteval.web.websocket import ws_manager
def _supplement_decision_logs(session) -> int:
"""Platform audit backfill for decision logs.
方案③的决策日志由 OpenClaw agent 上报LLM 自主,尽力而为)——异常路径
如卡死恢复后重试agent 可能跳过上报,导致决策过程页面为空。这里按评估
状态推导决策并补录:
- EXECUTING欠账completed < estimated补 execute_session所有会话
完成后补 start_analysis。
- COMPLETED历史评估/异常路径可能完全没有决策日志,回填 execute_session
(按 plan 时段逐条)+ start_analysis让旧报告也有决策过程可看。
只补"该类型缺失"的,不重复;且只记录状态,不改变 agent 的实际执行。
Returns:
补录的决策日志条数。
"""
from sqlmodel import select
from agenteval.intelligent_eval.decision_logs import create_decision_log
from agenteval.intelligent_eval.models import IntelligentEvalStatus
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalDecisionLogDB, IntelligentEvalSessionDB
evals = session.exec(
select(IntelligentEvalDB).where(
IntelligentEvalDB.status.in_(
[
IntelligentEvalStatus.EXECUTING.value,
IntelligentEvalStatus.COMPLETED.value,
]
)
)
).all()
added = 0
for ev in evals:
plan = ev.get_plan() if ev.plan else {}
estimated = plan.get("estimated_sessions", 0)
sessions = session.exec(select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.eval_id == ev.id)).all()
completed = sum(1 for s in sessions if s.status == "completed")
types = {
x.decision_type
for x in session.exec(
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
).all()
}
if ev.status == IntelligentEvalStatus.EXECUTING.value:
if "execute_session" not in types and completed < estimated:
create_decision_log(
ev.id,
"execute_session",
"平台兜底:时段欠账需执行会话",
"platform",
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
session,
)
added += 1
elif "start_analysis" not in types and sessions and completed >= estimated:
create_decision_log(
ev.id,
"start_analysis",
"平台兜底:所有会话已完成开始分析",
"platform",
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
session,
)
added += 1
elif ev.status == IntelligentEvalStatus.COMPLETED.value:
# 历史回填completed 评估决策日志全缺失时,按时段补 execute_session
if "execute_session" not in types:
slots = plan.get("time_distribution") or []
if slots:
for slot in slots:
create_decision_log(
ev.id,
"execute_session",
f"平台兜底:时段{slot.get('time_slot', '')}执行会话(历史回填)",
"platform",
{
"platform_supplemented": True,
"time_slot": slot.get("time_slot"),
"sessions": slot.get("sessions"),
"completed": completed,
"estimated": estimated,
},
session,
)
added += 1
else:
create_decision_log(
ev.id,
"execute_session",
"平台兜底:执行会话(历史回填)",
"platform",
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
session,
)
added += 1
if "start_analysis" not in types and sessions:
create_decision_log(
ev.id,
"start_analysis",
"平台兜底:所有会话已完成开始分析(历史回填)",
"platform",
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
session,
)
added += 1
return added
async def _trigger_intelligent_worker(nudge_eval_ids: Optional[list[str]] = None) -> bool:
"""Trigger OpenClaw's headless agent to run the intelligent-eval worker skill.
方案③(避免外部 IM channel平台只当"触发闹钟"——有 pending 任务时,
用 `docker exec openclaw-eval openclaw agent --agent main -m
"agenteval-intelligent-worker"` 唤醒 agentworker skill 由 OpenClaw agent
自主执行(决策/建会话/真实对话/上报,逻辑零改动)。`--deliver` 默认 false
因此不经 cron delivery channel。
ADR-0011 孤儿 agent 双管:
- 容器内命令包 `timeout 600`——agent 进程最多跑 10 分钟即被强杀;
- per-eval 触发冷却 10minworker_trigger 决策日志记录——agent 还在跑
的评估不重复触发,避免 60s 扫描节奏堆叠并发 agent。
nudge_eval_idsanalyst 兜底催促的评估(所有会话终态、无任务队列条目),
与候选任务评估合并去重后一同落账触发。
Returns:
True 若确实触发了 agent存在冷却期外的服务对象
"""
from agenteval.intelligent_eval.lifecycle import (
record_worker_triggers,
worker_trigger_candidates,
)
session = get_session()
try:
trigger_ids = sorted(set(worker_trigger_candidates(session)) | set(nudge_eval_ids or []))
if not trigger_ids:
return False
# 触发前落账(含后续失败也计入冷却),防止 agent 在跑期间重复触发
record_worker_triggers(session, trigger_ids)
finally:
session.close()
import subprocess
_logger = logging.getLogger("agenteval")
# 触发指令必须带"立即执行"语义:`openclaw agent` 无 cron state若只发
# skill 名agent 会按 worker skill 的"跨节拍"设计决策后等下一拍而死锁。
# 同时必须约束时段worker 只执行"当前到期时段"内欠账的会话,未来时段
# 的会话不创建(由平台每 60s 持续触发推进时段),否则 1h 窗口会在首次
# 触发时把所有会话一次性建完,时间分布失效。
worker_msg = (
"执行 agenteval-intelligent-worker skill立即完成当前任务不要等待下一节拍"
"仅执行当前时间对应时段time_distribution 中当前 offset 所在时段)内欠账的会话"
"——按评估 started_at 与当前时间精确判断当前时段,只创建该时段计划内的会话,"
"绝不创建未来时段的会话,未来时段到期后平台会再次触发你;"
"若所有会话已完成则调用 agenteval-intelligent-analyst skill 完成分析并生成报告。"
)
try:
proc = await asyncio.to_thread(
subprocess.run,
[
"docker",
"exec",
"openclaw-eval",
# ADR-0011容器内 timeout 强杀agent 最长存活 10min孤儿 agent 双管之一)
"timeout",
"600",
"openclaw",
"agent",
"--agent",
"main",
# 必须用独立 session`--agent main` 复用 main 持久 session多次触发
# 累积上下文缓存(~12 万 token后 LLM 不再执行 worker skill 的 API
# 步骤,直接幻觉输出(如"评估 pending_approval")而不取任务建会话。
"--session-id",
f"agenteval-worker-{int(time.time())}",
"-m",
worker_msg,
"--json",
],
capture_output=True,
text=True,
timeout=660, # 容器内 timeout 600 + 启动/回收余量
)
_logger.info("Worker 触发完成 exit=%s", proc.returncode)
if proc.returncode != 0:
_logger.warning("Worker 触发 stderr: %s", proc.stderr[-300:])
_record_worker_trigger_failure(f"exit={proc.returncode} stderr={proc.stderr[-200:]}")
except Exception as exc:
_logger.warning("Worker 触发失败(忽略): %s", exc)
_record_worker_trigger_failure(str(exc))
return True
def _record_worker_trigger_failure(error: str) -> None:
"""ADR-0011worker 触发失败落账到受影响评估(有待认领任务的评估)。"""
from agenteval.intelligent_eval.lifecycle import (
eval_ids_with_pending_worker_tasks,
record_trigger_failures,
)
session = get_session()
try:
eval_ids = eval_ids_with_pending_worker_tasks(session)
if eval_ids:
record_trigger_failures(session, channel="worker", eval_ids=eval_ids, error=error)
except Exception as exc:
logging.getLogger("agenteval").warning("触发失败落账失败(忽略): %s", exc)
finally:
session.close()
async def _trigger_intelligent_planner() -> bool:
"""Trigger OpenClaw's headless agent to run the planner skill.
方案③只自动化了 executingworker→ completedanalyst**缺少 planning
阶段的 planner 触发**——新建或被打回的评估会永远停在 planning。这里对
planning 状态评估触发 `agenteval-intelligent-planner` skillplanner 自会
拉取 planning 评估列表、读取四件套、产出粗计划并 PUT /plan 提交planner
skill 定义见 OpenClaw workspace skills。处理完评估离开 planning 后不再触发。
Returns:
True 若确实触发了 agent存在 planning 评估)。
"""
# ADR-0011触发前先落账 planner_trigger含后续失败也计入双闸
# 0 个 planning 评估时不触发。
from agenteval.intelligent_eval.lifecycle import record_planner_triggers
session = get_session()
try:
planning_count = record_planner_triggers(session)
finally:
session.close()
if planning_count == 0:
return False
import subprocess
_logger = logging.getLogger("agenteval")
# 同 worker`openclaw agent` 无 cron state须带"立即完成"语义,否则 planner
# 会"决策后等下一拍"而死锁。
planner_msg = (
"执行 agenteval-intelligent-planner skill立即完成当前任务不要等待下一节拍"
"为 planning 状态的智能评估读取输入、产出粗计划并提交平台审批。"
)
try:
proc = await asyncio.to_thread(
subprocess.run,
[
"docker",
"exec",
"openclaw-eval",
# ADR-0011容器内 timeout 强杀agent 最长存活 10min孤儿 agent 双管之一)
"timeout",
"600",
"openclaw",
"agent",
"--agent",
"main",
# 同 worker独立 session 避免 main 持久 session 上下文缓存污染
"--session-id",
f"agenteval-planner-{int(time.time())}",
"-m",
planner_msg,
"--json",
],
capture_output=True,
text=True,
timeout=660, # 容器内 timeout 600 + 启动/回收余量
)
_logger.info("Planner 触发完成 exit=%s", proc.returncode)
if proc.returncode != 0:
_logger.warning("Planner 触发 stderr: %s", proc.stderr[-300:])
_record_planner_trigger_failure(f"exit={proc.returncode} stderr={proc.stderr[-200:]}")
except Exception as exc:
_logger.warning("Planner 触发失败(忽略): %s", exc)
_record_planner_trigger_failure(str(exc))
return True
def _record_planner_trigger_failure(error: str) -> None:
"""ADR-0011planner 触发失败落账到所有 planning 评估。"""
from sqlmodel import select
from agenteval.intelligent_eval.lifecycle import record_trigger_failures
from agenteval.intelligent_eval.models import IntelligentEvalStatus
from agenteval.storage.db import IntelligentEvalDB
session = get_session()
try:
eval_ids = [
row.id
for row in session.exec(
select(IntelligentEvalDB).where(IntelligentEvalDB.status == IntelligentEvalStatus.PLANNING.value)
).all()
]
if eval_ids:
record_trigger_failures(session, channel="planner", eval_ids=eval_ids, error=error)
except Exception as exc:
logging.getLogger("agenteval").warning("触发失败落账失败(忽略): %s", exc)
finally:
session.close()
def _fire_and_forget(coro, name: str) -> None:
"""ADR-0011触发改为 fire-and-forget——agent 最长跑 10minawait 会把
60s 扫描节奏拖到 10min+。派生 asyncio task异常在完成回调中记录
(触发失败已由触发函数自身落账 trigger_failed"""
def _done(task: asyncio.Task) -> None:
if task.cancelled():
return
exc = task.exception()
if exc:
logging.getLogger("agenteval").warning("%s 触发任务异常(忽略): %s", name, exc)
asyncio.create_task(coro).add_done_callback(_done)
async def _intelligent_eval_scan_loop() -> None:
"""Scan intelligent evals, enqueue tasks, and trigger planner/worker.
v1.1.0 缺陷修复:`scan_and_enqueue_tasks` 此前没有调度点OpenClaw Worker
每分钟唤醒却永远取不到任务。平台启动后每 60s
① 有 planning 评估则触发 OpenClaw planner skill 产出粗计划planning→待审批
② 扫描 executing 的评估入队;③ 若有 pending 任务则触发 worker skill 执行会话/
分析(方案③,免外部 channel。失败不阻断下次循环继续
"""
_logger = logging.getLogger("agenteval")
while True:
try:
session = get_session()
try:
from agenteval.intelligent_eval.lifecycle import (
enforce_executing_ceiling,
enforce_planning_gates,
enforce_trigger_failure_gates,
expire_stale_running_sessions,
)
from agenteval.intelligent_eval.task_queue import (
requeue_stale_assigned_tasks,
scan_and_enqueue_tasks,
settle_tasks_for_finished_evals,
)
r = requeue_stale_assigned_tasks(session)
if r:
_logger.info("卡死恢复:%d 个 assigned 任务重新入队", r)
x = expire_stale_running_sessions(session)
if x:
_logger.info("会话过期:%d 个 running 会话 60 分钟无新轮次,置为 expired", x)
g = enforce_planning_gates(session)
if g:
_logger.info("planning 双闸:%d 个评估超限判失败", g)
c = enforce_executing_ceiling(session)
if c:
_logger.info("executing 兜底:%d 个评估超窗判失败", c)
f = enforce_trigger_failure_gates(session)
if f:
_logger.info("触发失败判死:%d 个评估连续触发失败超限判失败", f)
n = scan_and_enqueue_tasks(session)
if n:
_logger.info("智能评估扫描:入队 %d 个 Worker 任务", n)
# 清理:评估已结束(非 executing的 pending/assigned 任务回收为 completed
settled = settle_tasks_for_finished_evals(session)
if settled:
_logger.info("任务回收:%d 个已结束评估的任务标记完成", settled)
# 审计兜底agent 未上报决策日志时,平台按评估状态补录
added = _supplement_decision_logs(session)
if added:
_logger.info("决策日志兜底:补录 %d", added)
finally:
session.close()
except Exception as exc:
logging.getLogger("agenteval").warning("智能评估扫描失败(忽略): %s", exc)
# analyst 兜底ADR-0011末会话终态 10min 后平台催促 analyst 汇总报告,
# 催促落账后把评估交给 worker 触发worker 在全终态时会转 analyst 路径)
nudge_ids: list[str] = []
try:
from agenteval.intelligent_eval.lifecycle import (
evals_needing_analyst_nudge,
record_analyst_nudge,
)
session = get_session()
try:
nudge_ids = evals_needing_analyst_nudge(session)
for eval_id in nudge_ids:
record_analyst_nudge(session, eval_id)
finally:
session.close()
if nudge_ids:
logging.getLogger("agenteval").info("analyst 兜底:催促 %d 个评估汇总报告", len(nudge_ids))
except Exception as exc:
logging.getLogger("agenteval").warning("analyst 兜底失败(忽略): %s", exc)
# ADR-0011触发 fire-and-forget不阻塞 60s 扫描节奏agent 最长跑 10min
_fire_and_forget(_trigger_intelligent_worker(nudge_eval_ids=nudge_ids), "Worker")
_fire_and_forget(_trigger_intelligent_planner(), "Planner")
await asyncio.sleep(60)
@asynccontextmanager
async def lifespan(_: FastAPI):
init_db()
scan_task = asyncio.create_task(_intelligent_eval_scan_loop())
# 恢复耐久 Campaign/智能作业,并清理无法安全重放的中断运行(尽力而为,不阻断启动)。
try:
session = get_session()
try:
from agenteval.evaluation.campaign_runner import campaign_runtime
from agenteval.evaluation.intelligence_jobs import recover_campaign_intelligence_jobs
interrupted_jobs, resumed_jobs = recover_campaign_intelligence_jobs(session)
recovery = campaign_runtime.recover()
if recovery.interrupted_runs:
logging.getLogger("agenteval").warning(
"启动清理:%d 个中断的运行已标记为 failed", recovery.interrupted_runs
)
if interrupted_jobs:
logging.getLogger("agenteval").warning(
"启动清理:%d 条中断的分析/对比已标记为 failed",
interrupted_jobs,
)
if recovery.resumed_campaigns:
logging.getLogger("agenteval").warning(
"启动恢复:%d 个进行中的评估活动已续跑", recovery.resumed_campaigns
)
if resumed_jobs:
logging.getLogger("agenteval").warning(
"启动恢复:%d 条排队中的活动分析/对比已续跑",
resumed_jobs,
)
finally:
session.close()
except Exception as exc:
logging.getLogger("agenteval").warning("启动清理失败(忽略): %s", exc)
yield
# 优雅停止后台扫描,再停其他进程内任务。
scan_task.cancel()
try:
await scan_task
except asyncio.CancelledError:
pass
# 优雅停止所有进程内任务:先停活动调度循环,再停在跑的评测运行,
# 最后停活动智能作业(分析 / 周期对比)和 judge 复核。
try:
from agenteval.evaluation.campaign_runner import campaign_runtime
from agenteval.evaluation.intelligence_jobs import shutdown_campaign_intelligence_jobs
from agenteval.exploration.judge import judge_registry
from agenteval.web.routers.runs import run_registry
await campaign_runtime.shutdown()
await run_registry.shutdown_all()
await shutdown_campaign_intelligence_jobs()
await judge_registry.shutdown_all()
except Exception as exc:
logging.getLogger("agenteval").warning("活动调度停止失败(忽略): %s", exc)
app = FastAPI(
title="AgentEvalTool",
description="智能体质量评估工具集平台 Web API",
version=get_version(),
lifespan=lifespan,
)
settings = get_settings()
app.add_middleware(
CORSMiddleware,
allow_origins=settings.allowed_origins,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
_api_deps = [Depends(require_api_key)]
# Login endpoints must stay open — they are how the client obtains credentials.
app.include_router(auth.router, prefix="/api/auth", tags=["auth"])
app.include_router(targets.router, prefix="/api/targets", tags=["targets"], dependencies=_api_deps)
app.include_router(scenarios.router, prefix="/api/scenarios", tags=["scenarios"], dependencies=_api_deps)
app.include_router(runs.router, prefix="/api/runs", tags=["runs"], dependencies=_api_deps)
app.include_router(campaigns.router, prefix="/api/campaigns", tags=["campaigns"], dependencies=_api_deps)
app.include_router(exploration.router, prefix="/api/exploration", tags=["exploration"], dependencies=_api_deps)
app.include_router(
intelligent_evals.router, prefix="/api/intelligent-evals", tags=["intelligent-evals"], dependencies=_api_deps
)
app.include_router(reports.router, prefix="/api/reports", tags=["reports"], dependencies=_api_deps)
app.include_router(stats.router, prefix="/api/stats", tags=["stats"], dependencies=_api_deps)
app.include_router(files.router, prefix="/api/files", tags=["files"], dependencies=_api_deps)
app.include_router(
model_configs.router,
prefix="/api/model-configs",
tags=["model-configs"],
dependencies=_api_deps,
)
@app.websocket("/openclaw")
async def openclaw_ws_root(websocket: WebSocket):
url = proxy.get_ws_upstream()
if websocket.query_params:
url += f"?{websocket.query_params}"
await proxy.ws_bridge(websocket, url)
@app.websocket("/openclaw/{path:path}")
async def openclaw_ws_proxy(websocket: WebSocket, path: str):
url = f"{proxy.get_ws_upstream()}/{path}"
if websocket.query_params:
url += f"?{websocket.query_params}"
await proxy.ws_bridge(websocket, url)
app.include_router(proxy.router, prefix="/openclaw", tags=["proxy"])
_default_dist = Path(__file__).resolve().parent.parent.parent.parent / "frontend" / "web" / "dist"
WEB_DIST = Path(settings.frontend_dist_path) if settings.frontend_dist_path else _default_dist
if WEB_DIST.exists():
from fastapi.staticfiles import StaticFiles
app.mount("/assets", StaticFiles(directory=WEB_DIST / "assets"), name="assets")
@app.get("/api/health", tags=["health"])
def health() -> dict:
return {"status": "ok", **get_build_info()}
@app.websocket("/ws/runs/{run_id}")
async def websocket_run_progress(websocket: WebSocket, run_id: str):
await ws_manager.connect(run_id, websocket)
try:
while True:
await websocket.receive_text()
except WebSocketDisconnect:
ws_manager.disconnect(run_id, websocket)
@app.get("/{full_path:path}")
def serve_spa(full_path: str):
"""Serve the React SPA for all non-API routes."""
index_file = WEB_DIST / "index.html"
if index_file.exists():
return FileResponse(index_file)
return JSONResponse({"detail": "frontend not built"}, status_code=404)