"""Alert rules and notifications for cron pool (告警规则和通知). Alert rules: - Pool utilization > 90% for 10 minutes - Task backlog > 50 - Stuck rate > 10% Notifications: - Log alerts - Optional webhook notifications """ import logging from datetime import datetime from typing import Optional import httpx from sqlmodel import Field, Session, SQLModel, select from agenteval.intelligent_eval.metrics import ( calculate_pool_utilization, calculate_stuck_rate, calculate_task_backlog, ) from agenteval.storage.db import utc_now _logger = logging.getLogger("agenteval") class AlertHistoryDB(SQLModel, table=True): """Alert history database record.""" __tablename__ = "cron_pool_alert_history" id: Optional[str] = Field(default=None, primary_key=True) alert_type: str = Field(index=True) # pool_utilization / task_backlog / stuck_rate severity: str = Field(index=True) # warning / critical message: str metric_value: float threshold: float created_at: datetime = Field(default_factory=utc_now) resolved_at: Optional[datetime] = None webhook_sent: bool = False class AlertRule: """Alert rule definition.""" def __init__( self, name: str, metric_func, threshold: float, severity: str, duration_minutes: int = 0, ): self.name = name self.metric_func = metric_func self.threshold = threshold self.severity = severity self.duration_minutes = duration_minutes self.triggered_at: Optional[datetime] = None class AlertManager: """Manages alert rules and notifications.""" def __init__( self, session: Session, webhook_url: Optional[str] = None, openclaw_client=None, ): self.session = session self.webhook_url = webhook_url self.openclaw_client = openclaw_client self.rules = [ AlertRule( name="pool_utilization", metric_func=calculate_pool_utilization, threshold=0.9, severity="warning", duration_minutes=10, ), AlertRule( name="task_backlog", metric_func=calculate_task_backlog, threshold=50, severity="warning", duration_minutes=0, ), AlertRule( name="stuck_rate", metric_func=calculate_stuck_rate, threshold=0.1, severity="critical", duration_minutes=0, ), ] def check_rules(self) -> list[AlertHistoryDB]: """Check all alert rules and create alerts if triggered. Returns: List of newly created alerts """ alerts = [] for rule in self.rules: metric_value = rule.metric_func(self.session) # Check if threshold exceeded if metric_value > rule.threshold: # Check duration requirement if rule.duration_minutes > 0: if rule.triggered_at is None: rule.triggered_at = utc_now() continue duration = (utc_now() - rule.triggered_at).total_seconds() / 60 if duration < rule.duration_minutes: continue else: rule.triggered_at = utc_now() # Create alert alert = self._create_alert(rule, metric_value) alerts.append(alert) # Send webhook notification if self.webhook_url: self._send_webhook(alert) # Note: alert→auto-scale link is invoked from the async router # via ``AlertManager.maybe_autoscale`` so we stay out of nested # event-loop territory. else: # Reset trigger time rule.triggered_at = None return alerts def _create_alert(self, rule: AlertRule, metric_value: float) -> AlertHistoryDB: """Create an alert history record.""" message = f"{rule.name}: {metric_value:.2f} exceeds threshold {rule.threshold}" alert = AlertHistoryDB( id=f"alert-{utc_now().timestamp()}", alert_type=rule.name, severity=rule.severity, message=message, metric_value=metric_value, threshold=rule.threshold, ) self.session.add(alert) self.session.commit() self.session.refresh(alert) _logger.warning(f"Alert triggered: {message}") return alert def _send_webhook(self, alert: AlertHistoryDB, max_retries: int = 3) -> None: """Send webhook with retry and dedupe. P1 真问题修复（§6.2）: - **Dedupe**: if ``alert.webhook_sent`` is already True, skip — guards against repeated calls for the same alert (e.g. consecutive ``check_alerts`` ticks). - **Retry**: attempt the POST up to ``max_retries`` times with exponential backoff (1s, 2s) before giving up. Persist ``webhook_sent=True`` only on success. """ if not self.webhook_url: return if alert.webhook_sent: return payload = { "alert_id": alert.id, "alert_type": alert.alert_type, "severity": alert.severity, "message": alert.message, "metric_value": alert.metric_value, "threshold": alert.threshold, "timestamp": alert.created_at.isoformat(), } import time last_err: Exception | None = None for attempt in range(max_retries): try: response = httpx.post(self.webhook_url, json=payload, timeout=5.0) response.raise_for_status() alert.webhook_sent = True self.session.commit() _logger.info(f"Webhook sent for alert {alert.id} (attempt {attempt + 1})") return except Exception as e: last_err = e if attempt < max_retries - 1: time.sleep(2 ** attempt) _logger.error(f"Failed to send webhook for alert {alert.id} after {max_retries} attempts: {last_err}") def get_alert_history(self, limit: int = 100) -> list[AlertHistoryDB]: """Get alert history. Returns: List of alerts, ordered by created_at descending """ alerts = self.session.exec( select(AlertHistoryDB) .order_by(AlertHistoryDB.created_at.desc()) .limit(limit) ).all() return list(alerts) def get_unresolved_alerts(self) -> list[AlertHistoryDB]: """Get unresolved alerts. Returns: List of unresolved alerts """ alerts = self.session.exec( select(AlertHistoryDB).where(AlertHistoryDB.resolved_at.is_(None)) ).all() return list(alerts) def resolve_alert(self, alert_id: str) -> bool: """Resolve an alert. Returns: True if alert was resolved, False if not found """ alert = self.session.get(AlertHistoryDB, alert_id) if alert is None: return False alert.resolved_at = utc_now() self.session.commit() return True async def maybe_autoscale(self, alert: AlertHistoryDB) -> None: """Async entry point for the alert→auto-scale link (T7). Called by the router after ``check_rules``. Best-effort: failures are logged, not raised. ``cron_pool.scale_up`` itself caps at ``MAX_POOL_SIZE``, so repeated invocations are safe. """ if self.openclaw_client is None: return try: from agenteval.intelligent_eval import cron_pool as _cp await _cp.scale_up(1, self.session, self.openclaw_client) except Exception as e: _logger.error(f"Auto-scale-up on alert {alert.id} failed: {e}")