AgentEvalTool/backend/agenteval/web/routers/openclaw_cron_pool.py
sinohqb 2e7d419f05 feat(intelligent-eval): implement monitoring and alerting (ticket 07)
- Add metrics.py with pool utilization, task backlog, stuck rate, avg processing time, eval completion rate
- Add alerts.py with alert rules (pool utilization > 90%, task backlog > 50, stuck rate > 10%)
- Implement alert history and webhook notifications
- Add metrics and alerts APIs
- Add database migration for alert history table
- Add 11 unit tests for metrics, 10 unit tests for alerts, 8 integration tests
- Update migration tests to include new alert history table

All 853 tests passing.
2026-08-12 10:41:15 +08:00

186 lines
5.7 KiB
Python

"""API routes for OpenClaw cron pool management."""
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlmodel import Session, select
from agenteval.intelligent_eval import cron_pool
from agenteval.intelligent_eval.openclaw_client import OpenClawClient
from agenteval.storage.db import OpenClawCronPoolDB, utc_now
from agenteval.web.deps import get_db
router = APIRouter()
class ScaleRequest(BaseModel):
target_size: int = Field(ge=1, le=50)
class HeartbeatRequest(BaseModel):
status: str # idle / busy
current_eval_id: str | None = None
@router.get("/cron-pool")
async def get_cron_pool_status(session: Session = Depends(get_db)) -> dict:
"""Get cron pool status."""
status = cron_pool.get_pool_status(session)
return {"pool": status}
@router.post("/cron-pool/scale")
async def scale_cron_pool(request: ScaleRequest, session: Session = Depends(get_db)) -> dict:
"""Manually scale cron pool to target size."""
current_status = cron_pool.get_pool_status(session)
current_size = current_status["total"]
target_size = request.target_size
client = OpenClawClient()
if target_size > current_size:
# Scale up
count = target_size - current_size
created = await cron_pool.scale_up(count, session, client)
return {"success": True, "scaled_up": created, "current_size": current_size + created}
elif target_size < current_size:
# Scale down
count = current_size - target_size
deleted = await cron_pool.scale_down(count, session, client)
return {"success": True, "scaled_down": deleted, "current_size": current_size - deleted}
else:
return {"success": True, "current_size": current_size, "message": "already at target size"}
@router.post("/cron-pool/sync")
async def sync_cron_states(session: Session = Depends(get_db)) -> dict:
"""Sync cron states from OpenClaw to platform DB."""
client = OpenClawClient()
synced = await cron_pool.sync_cron_states(session, client)
return {"success": True, "synced": synced}
@router.post("/cron-pool/auto-scale")
async def auto_scale_pool(session: Session = Depends(get_db)) -> dict:
"""Trigger auto-scaling based on current load."""
client = OpenClawClient()
scaled_up, scaled_down = await cron_pool.auto_scale(session, client)
return {
"success": True,
"scaled_up": scaled_up,
"scaled_down": scaled_down,
}
@router.post("/crons/{cron_id}/heartbeat")
async def report_heartbeat(
cron_id: str,
request: HeartbeatRequest,
session: Session = Depends(get_db),
) -> dict:
"""Report cron heartbeat.
Updates the cron's last_active_at timestamp and current status.
"""
# Find cron by openclaw_cron_id
cron = session.exec(
select(OpenClawCronPoolDB).where(OpenClawCronPoolDB.openclaw_cron_id == cron_id)
).first()
if cron is None:
raise HTTPException(status_code=404, detail=f"cron {cron_id} not found")
# Update heartbeat
cron.last_active_at = utc_now()
cron.status = request.status
cron.current_eval_id = request.current_eval_id
cron.updated_at = utc_now()
session.commit()
return {"success": True}
@router.get("/cron-pool/metrics")
async def get_cron_pool_metrics(session: Session = Depends(get_db)) -> dict:
"""Get cron pool metrics."""
from agenteval.intelligent_eval.metrics import get_all_metrics
metrics = get_all_metrics(session)
return {"metrics": metrics}
@router.post("/cron-pool/check-alerts")
async def check_alerts(session: Session = Depends(get_db)) -> dict:
"""Check alert rules and create alerts if triggered."""
from agenteval.intelligent_eval.alerts import AlertManager
manager = AlertManager(session)
alerts = manager.check_rules()
return {
"success": True,
"alerts_triggered": len(alerts),
"alerts": [
{
"id": alert.id,
"alert_type": alert.alert_type,
"severity": alert.severity,
"message": alert.message,
"metric_value": alert.metric_value,
"threshold": alert.threshold,
"created_at": alert.created_at.isoformat(),
}
for alert in alerts
],
}
@router.get("/cron-pool/alerts")
async def get_alert_history(
limit: int = 100,
unresolved_only: bool = False,
session: Session = Depends(get_db),
) -> dict:
"""Get alert history."""
from agenteval.intelligent_eval.alerts import AlertManager
manager = AlertManager(session)
if unresolved_only:
alerts = manager.get_unresolved_alerts()
else:
alerts = manager.get_alert_history(limit=limit)
return {
"alerts": [
{
"id": alert.id,
"alert_type": alert.alert_type,
"severity": alert.severity,
"message": alert.message,
"metric_value": alert.metric_value,
"threshold": alert.threshold,
"created_at": alert.created_at.isoformat(),
"resolved_at": alert.resolved_at.isoformat() if alert.resolved_at else None,
"webhook_sent": alert.webhook_sent,
}
for alert in alerts
]
}
@router.post("/cron-pool/alerts/{alert_id}/resolve")
async def resolve_alert(alert_id: str, session: Session = Depends(get_db)) -> dict:
"""Resolve an alert."""
from agenteval.intelligent_eval.alerts import AlertManager
manager = AlertManager(session)
resolved = manager.resolve_alert(alert_id)
if not resolved:
raise HTTPException(status_code=404, detail=f"alert {alert_id} not found")
return {"success": True}