AgentEvalTool/backend/agenteval/intelligent_eval/alerts.py

"""Alert rules and notifications for cron pool (告警规则和通知).

Alert rules:
- Pool utilization > 90% for 10 minutes
- Task backlog > 50
- Stuck rate > 10%

Notifications:
- Log alerts
- Optional webhook notifications
"""

import logging
from datetime import datetime
from typing import Optional

import httpx
from sqlmodel import Field, Session, SQLModel, select

from agenteval.intelligent_eval.metrics import (
    calculate_pool_utilization,
    calculate_stuck_rate,
    calculate_task_backlog,
)
from agenteval.storage.db import utc_now

_logger = logging.getLogger("agenteval")


class AlertHistoryDB(SQLModel, table=True):
    """Alert history database record."""

    __tablename__ = "cron_pool_alert_history"

    id: Optional[str] = Field(default=None, primary_key=True)
    alert_type: str = Field(index=True)  # pool_utilization / task_backlog / stuck_rate
    severity: str = Field(index=True)  # warning / critical
    message: str
    metric_value: float
    threshold: float
    created_at: datetime = Field(default_factory=utc_now)
    resolved_at: Optional[datetime] = None
    webhook_sent: bool = False


class AlertRule:
    """Alert rule definition."""

    def __init__(
        self,
        name: str,
        metric_func,
        threshold: float,
        severity: str,
        duration_minutes: int = 0,
    ):
        self.name = name
        self.metric_func = metric_func
        self.threshold = threshold
        self.severity = severity
        self.duration_minutes = duration_minutes
        self.triggered_at: Optional[datetime] = None


class AlertManager:
    """Manages alert rules and notifications."""

    def __init__(self, session: Session, webhook_url: Optional[str] = None):
        self.session = session
        self.webhook_url = webhook_url
        self.rules = [
            AlertRule(
                name="pool_utilization",
                metric_func=calculate_pool_utilization,
                threshold=0.9,
                severity="warning",
                duration_minutes=10,
            ),
            AlertRule(
                name="task_backlog",
                metric_func=calculate_task_backlog,
                threshold=50,
                severity="warning",
                duration_minutes=0,
            ),
            AlertRule(
                name="stuck_rate",
                metric_func=calculate_stuck_rate,
                threshold=0.1,
                severity="critical",
                duration_minutes=0,
            ),
        ]

    def check_rules(self) -> list[AlertHistoryDB]:
        """Check all alert rules and create alerts if triggered.

        Returns:
            List of newly created alerts
        """
        alerts = []

        for rule in self.rules:
            metric_value = rule.metric_func(self.session)

            # Check if threshold exceeded
            if metric_value > rule.threshold:
                # Check duration requirement
                if rule.duration_minutes > 0:
                    if rule.triggered_at is None:
                        rule.triggered_at = utc_now()
                        continue

                    duration = (utc_now() - rule.triggered_at).total_seconds() / 60
                    if duration < rule.duration_minutes:
                        continue
                else:
                    rule.triggered_at = utc_now()

                # Create alert
                alert = self._create_alert(rule, metric_value)
                alerts.append(alert)

                # Send webhook notification
                if self.webhook_url:
                    self._send_webhook(alert)
            else:
                # Reset trigger time
                rule.triggered_at = None

        return alerts

    def _create_alert(self, rule: AlertRule, metric_value: float) -> AlertHistoryDB:
        """Create an alert history record."""
        message = f"{rule.name}: {metric_value:.2f} exceeds threshold {rule.threshold}"

        alert = AlertHistoryDB(
            id=f"alert-{utc_now().timestamp()}",
            alert_type=rule.name,
            severity=rule.severity,
            message=message,
            metric_value=metric_value,
            threshold=rule.threshold,
        )

        self.session.add(alert)
        self.session.commit()
        self.session.refresh(alert)

        _logger.warning(f"Alert triggered: {message}")

        return alert

    def _send_webhook(self, alert: AlertHistoryDB, max_retries: int = 3) -> None:
        """Send webhook with retry and dedupe.

        P1 真问题修复（§6.2）:
          - **Dedupe**: if ``alert.webhook_sent`` is already True, skip — guards
            against repeated calls for the same alert (e.g. consecutive
            ``check_alerts`` ticks).
          - **Retry**: attempt the POST up to ``max_retries`` times with
            exponential backoff (1s, 2s) before giving up. Persist
            ``webhook_sent=True`` only on success.
        """
        if not self.webhook_url:
            return
        if alert.webhook_sent:
            return

        payload = {
            "alert_id": alert.id,
            "alert_type": alert.alert_type,
            "severity": alert.severity,
            "message": alert.message,
            "metric_value": alert.metric_value,
            "threshold": alert.threshold,
            "timestamp": alert.created_at.isoformat(),
        }

        import time

        last_err: Exception | None = None
        for attempt in range(max_retries):
            try:
                response = httpx.post(self.webhook_url, json=payload, timeout=5.0)
                response.raise_for_status()
                alert.webhook_sent = True
                self.session.commit()
                _logger.info(f"Webhook sent for alert {alert.id} (attempt {attempt + 1})")
                return
            except Exception as e:
                last_err = e
                if attempt < max_retries - 1:
                    time.sleep(2 ** attempt)

        _logger.error(f"Failed to send webhook for alert {alert.id} after {max_retries} attempts: {last_err}")

    def get_alert_history(self, limit: int = 100) -> list[AlertHistoryDB]:
        """Get alert history.

        Returns:
            List of alerts, ordered by created_at descending
        """
        alerts = self.session.exec(
            select(AlertHistoryDB)
            .order_by(AlertHistoryDB.created_at.desc())
            .limit(limit)
        ).all()
        return list(alerts)

    def get_unresolved_alerts(self) -> list[AlertHistoryDB]:
        """Get unresolved alerts.

        Returns:
            List of unresolved alerts
        """
        alerts = self.session.exec(
            select(AlertHistoryDB).where(AlertHistoryDB.resolved_at.is_(None))
        ).all()
        return list(alerts)

    def resolve_alert(self, alert_id: str) -> bool:
        """Resolve an alert.

        Returns:
            True if alert was resolved, False if not found
        """
        alert = self.session.get(AlertHistoryDB, alert_id)
        if alert is None:
            return False

        alert.resolved_at = utc_now()
        self.session.commit()
        return True