- Add metrics.py with pool utilization, task backlog, stuck rate, avg processing time, eval completion rate - Add alerts.py with alert rules (pool utilization > 90%, task backlog > 50, stuck rate > 10%) - Implement alert history and webhook notifications - Add metrics and alerts APIs - Add database migration for alert history table - Add 11 unit tests for metrics, 10 unit tests for alerts, 8 integration tests - Update migration tests to include new alert history table All 853 tests passing.
186 lines
5.7 KiB
Python
186 lines
5.7 KiB
Python
"""API routes for OpenClaw cron pool management."""
|
|
|
|
from datetime import datetime
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException
|
|
from pydantic import BaseModel, Field
|
|
from sqlmodel import Session, select
|
|
|
|
from agenteval.intelligent_eval import cron_pool
|
|
from agenteval.intelligent_eval.openclaw_client import OpenClawClient
|
|
from agenteval.storage.db import OpenClawCronPoolDB, utc_now
|
|
from agenteval.web.deps import get_db
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
class ScaleRequest(BaseModel):
|
|
target_size: int = Field(ge=1, le=50)
|
|
|
|
|
|
class HeartbeatRequest(BaseModel):
|
|
status: str # idle / busy
|
|
current_eval_id: str | None = None
|
|
|
|
|
|
@router.get("/cron-pool")
|
|
async def get_cron_pool_status(session: Session = Depends(get_db)) -> dict:
|
|
"""Get cron pool status."""
|
|
status = cron_pool.get_pool_status(session)
|
|
return {"pool": status}
|
|
|
|
|
|
@router.post("/cron-pool/scale")
|
|
async def scale_cron_pool(request: ScaleRequest, session: Session = Depends(get_db)) -> dict:
|
|
"""Manually scale cron pool to target size."""
|
|
current_status = cron_pool.get_pool_status(session)
|
|
current_size = current_status["total"]
|
|
target_size = request.target_size
|
|
|
|
client = OpenClawClient()
|
|
|
|
if target_size > current_size:
|
|
# Scale up
|
|
count = target_size - current_size
|
|
created = await cron_pool.scale_up(count, session, client)
|
|
return {"success": True, "scaled_up": created, "current_size": current_size + created}
|
|
elif target_size < current_size:
|
|
# Scale down
|
|
count = current_size - target_size
|
|
deleted = await cron_pool.scale_down(count, session, client)
|
|
return {"success": True, "scaled_down": deleted, "current_size": current_size - deleted}
|
|
else:
|
|
return {"success": True, "current_size": current_size, "message": "already at target size"}
|
|
|
|
|
|
@router.post("/cron-pool/sync")
|
|
async def sync_cron_states(session: Session = Depends(get_db)) -> dict:
|
|
"""Sync cron states from OpenClaw to platform DB."""
|
|
client = OpenClawClient()
|
|
synced = await cron_pool.sync_cron_states(session, client)
|
|
return {"success": True, "synced": synced}
|
|
|
|
|
|
@router.post("/cron-pool/auto-scale")
|
|
async def auto_scale_pool(session: Session = Depends(get_db)) -> dict:
|
|
"""Trigger auto-scaling based on current load."""
|
|
client = OpenClawClient()
|
|
scaled_up, scaled_down = await cron_pool.auto_scale(session, client)
|
|
return {
|
|
"success": True,
|
|
"scaled_up": scaled_up,
|
|
"scaled_down": scaled_down,
|
|
}
|
|
|
|
|
|
@router.post("/crons/{cron_id}/heartbeat")
|
|
async def report_heartbeat(
|
|
cron_id: str,
|
|
request: HeartbeatRequest,
|
|
session: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Report cron heartbeat.
|
|
|
|
Updates the cron's last_active_at timestamp and current status.
|
|
"""
|
|
# Find cron by openclaw_cron_id
|
|
cron = session.exec(
|
|
select(OpenClawCronPoolDB).where(OpenClawCronPoolDB.openclaw_cron_id == cron_id)
|
|
).first()
|
|
|
|
if cron is None:
|
|
raise HTTPException(status_code=404, detail=f"cron {cron_id} not found")
|
|
|
|
# Update heartbeat
|
|
cron.last_active_at = utc_now()
|
|
cron.status = request.status
|
|
cron.current_eval_id = request.current_eval_id
|
|
cron.updated_at = utc_now()
|
|
|
|
session.commit()
|
|
|
|
return {"success": True}
|
|
|
|
|
|
@router.get("/cron-pool/metrics")
|
|
async def get_cron_pool_metrics(session: Session = Depends(get_db)) -> dict:
|
|
"""Get cron pool metrics."""
|
|
from agenteval.intelligent_eval.metrics import get_all_metrics
|
|
|
|
metrics = get_all_metrics(session)
|
|
return {"metrics": metrics}
|
|
|
|
|
|
@router.post("/cron-pool/check-alerts")
|
|
async def check_alerts(session: Session = Depends(get_db)) -> dict:
|
|
"""Check alert rules and create alerts if triggered."""
|
|
from agenteval.intelligent_eval.alerts import AlertManager
|
|
|
|
manager = AlertManager(session)
|
|
alerts = manager.check_rules()
|
|
|
|
return {
|
|
"success": True,
|
|
"alerts_triggered": len(alerts),
|
|
"alerts": [
|
|
{
|
|
"id": alert.id,
|
|
"alert_type": alert.alert_type,
|
|
"severity": alert.severity,
|
|
"message": alert.message,
|
|
"metric_value": alert.metric_value,
|
|
"threshold": alert.threshold,
|
|
"created_at": alert.created_at.isoformat(),
|
|
}
|
|
for alert in alerts
|
|
],
|
|
}
|
|
|
|
|
|
@router.get("/cron-pool/alerts")
|
|
async def get_alert_history(
|
|
limit: int = 100,
|
|
unresolved_only: bool = False,
|
|
session: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Get alert history."""
|
|
from agenteval.intelligent_eval.alerts import AlertManager
|
|
|
|
manager = AlertManager(session)
|
|
|
|
if unresolved_only:
|
|
alerts = manager.get_unresolved_alerts()
|
|
else:
|
|
alerts = manager.get_alert_history(limit=limit)
|
|
|
|
return {
|
|
"alerts": [
|
|
{
|
|
"id": alert.id,
|
|
"alert_type": alert.alert_type,
|
|
"severity": alert.severity,
|
|
"message": alert.message,
|
|
"metric_value": alert.metric_value,
|
|
"threshold": alert.threshold,
|
|
"created_at": alert.created_at.isoformat(),
|
|
"resolved_at": alert.resolved_at.isoformat() if alert.resolved_at else None,
|
|
"webhook_sent": alert.webhook_sent,
|
|
}
|
|
for alert in alerts
|
|
]
|
|
}
|
|
|
|
|
|
@router.post("/cron-pool/alerts/{alert_id}/resolve")
|
|
async def resolve_alert(alert_id: str, session: Session = Depends(get_db)) -> dict:
|
|
"""Resolve an alert."""
|
|
from agenteval.intelligent_eval.alerts import AlertManager
|
|
|
|
manager = AlertManager(session)
|
|
resolved = manager.resolve_alert(alert_id)
|
|
|
|
if not resolved:
|
|
raise HTTPException(status_code=404, detail=f"alert {alert_id} not found")
|
|
|
|
return {"success": True}
|