"""Alert rules and notifications for cron pool (告警规则和通知). Alert rules: - Pool utilization > 90% for 10 minutes - Task backlog > 50 - Stuck rate > 10% Notifications: - Log alerts - Optional webhook notifications """ import logging from datetime import datetime from typing import Optional import httpx from sqlmodel import Field, Session, SQLModel, select from agenteval.intelligent_eval.metrics import ( calculate_pool_utilization, calculate_stuck_rate, calculate_task_backlog, ) from agenteval.storage.db import utc_now _logger = logging.getLogger("agenteval") class AlertHistoryDB(SQLModel, table=True): """Alert history database record.""" __tablename__ = "cron_pool_alert_history" id: Optional[str] = Field(default=None, primary_key=True) alert_type: str = Field(index=True) # pool_utilization / task_backlog / stuck_rate severity: str = Field(index=True) # warning / critical message: str metric_value: float threshold: float created_at: datetime = Field(default_factory=utc_now) resolved_at: Optional[datetime] = None webhook_sent: bool = False class AlertRule: """Alert rule definition.""" def __init__( self, name: str, metric_func, threshold: float, severity: str, duration_minutes: int = 0, ): self.name = name self.metric_func = metric_func self.threshold = threshold self.severity = severity self.duration_minutes = duration_minutes self.triggered_at: Optional[datetime] = None class AlertManager: """Manages alert rules and notifications.""" def __init__(self, session: Session, webhook_url: Optional[str] = None): self.session = session self.webhook_url = webhook_url self.rules = [ AlertRule( name="pool_utilization", metric_func=calculate_pool_utilization, threshold=0.9, severity="warning", duration_minutes=10, ), AlertRule( name="task_backlog", metric_func=calculate_task_backlog, threshold=50, severity="warning", duration_minutes=0, ), AlertRule( name="stuck_rate", metric_func=calculate_stuck_rate, threshold=0.1, severity="critical", duration_minutes=0, ), ] def check_rules(self) -> list[AlertHistoryDB]: """Check all alert rules and create alerts if triggered. Returns: List of newly created alerts """ alerts = [] for rule in self.rules: metric_value = rule.metric_func(self.session) # Check if threshold exceeded if metric_value > rule.threshold: # Check duration requirement if rule.duration_minutes > 0: if rule.triggered_at is None: rule.triggered_at = utc_now() continue duration = (utc_now() - rule.triggered_at).total_seconds() / 60 if duration < rule.duration_minutes: continue else: rule.triggered_at = utc_now() # Create alert alert = self._create_alert(rule, metric_value) alerts.append(alert) # Send webhook notification if self.webhook_url: self._send_webhook(alert) else: # Reset trigger time rule.triggered_at = None return alerts def _create_alert(self, rule: AlertRule, metric_value: float) -> AlertHistoryDB: """Create an alert history record.""" message = f"{rule.name}: {metric_value:.2f} exceeds threshold {rule.threshold}" alert = AlertHistoryDB( id=f"alert-{utc_now().timestamp()}", alert_type=rule.name, severity=rule.severity, message=message, metric_value=metric_value, threshold=rule.threshold, ) self.session.add(alert) self.session.commit() self.session.refresh(alert) _logger.warning(f"Alert triggered: {message}") return alert def _send_webhook(self, alert: AlertHistoryDB) -> None: """Send webhook notification.""" if not self.webhook_url: return try: payload = { "alert_id": alert.id, "alert_type": alert.alert_type, "severity": alert.severity, "message": alert.message, "metric_value": alert.metric_value, "threshold": alert.threshold, "timestamp": alert.created_at.isoformat(), } response = httpx.post(self.webhook_url, json=payload, timeout=5.0) response.raise_for_status() alert.webhook_sent = True self.session.commit() _logger.info(f"Webhook sent for alert {alert.id}") except Exception as e: _logger.error(f"Failed to send webhook: {e}") def get_alert_history(self, limit: int = 100) -> list[AlertHistoryDB]: """Get alert history. Returns: List of alerts, ordered by created_at descending """ alerts = self.session.exec( select(AlertHistoryDB) .order_by(AlertHistoryDB.created_at.desc()) .limit(limit) ).all() return list(alerts) def get_unresolved_alerts(self) -> list[AlertHistoryDB]: """Get unresolved alerts. Returns: List of unresolved alerts """ alerts = self.session.exec( select(AlertHistoryDB).where(AlertHistoryDB.resolved_at.is_(None)) ).all() return list(alerts) def resolve_alert(self, alert_id: str) -> bool: """Resolve an alert. Returns: True if alert was resolved, False if not found """ alert = self.session.get(AlertHistoryDB, alert_id) if alert is None: return False alert.resolved_at = utc_now() self.session.commit() return True