6d32653675
All checks were successful
CI / test (push) Successful in 3m56s
AlertManager gains an optional openclaw_client. check_alerts records each newly created alert and AlertManager.maybe_autoscale (called from the async router for each alert) invokes cron_pool.scale_up(1). scale_up itself caps at MAX_POOL_SIZE so repeated invocations are safe. Removed the xfail guard in test_alert_autoscale_link; rewrote the test to use task_backlog (duration_minutes=0) so a single check_alerts call fires an alert and triggers auto-scale.
260 lines
8.0 KiB
Python
260 lines
8.0 KiB
Python
"""Alert rules and notifications for cron pool (告警规则和通知).
|
||
|
||
Alert rules:
|
||
- Pool utilization > 90% for 10 minutes
|
||
- Task backlog > 50
|
||
- Stuck rate > 10%
|
||
|
||
Notifications:
|
||
- Log alerts
|
||
- Optional webhook notifications
|
||
"""
|
||
|
||
import logging
|
||
from datetime import datetime
|
||
from typing import Optional
|
||
|
||
import httpx
|
||
from sqlmodel import Field, Session, SQLModel, select
|
||
|
||
from agenteval.intelligent_eval.metrics import (
|
||
calculate_pool_utilization,
|
||
calculate_stuck_rate,
|
||
calculate_task_backlog,
|
||
)
|
||
from agenteval.storage.db import utc_now
|
||
|
||
_logger = logging.getLogger("agenteval")
|
||
|
||
|
||
class AlertHistoryDB(SQLModel, table=True):
|
||
"""Alert history database record."""
|
||
|
||
__tablename__ = "cron_pool_alert_history"
|
||
|
||
id: Optional[str] = Field(default=None, primary_key=True)
|
||
alert_type: str = Field(index=True) # pool_utilization / task_backlog / stuck_rate
|
||
severity: str = Field(index=True) # warning / critical
|
||
message: str
|
||
metric_value: float
|
||
threshold: float
|
||
created_at: datetime = Field(default_factory=utc_now)
|
||
resolved_at: Optional[datetime] = None
|
||
webhook_sent: bool = False
|
||
|
||
|
||
class AlertRule:
|
||
"""Alert rule definition."""
|
||
|
||
def __init__(
|
||
self,
|
||
name: str,
|
||
metric_func,
|
||
threshold: float,
|
||
severity: str,
|
||
duration_minutes: int = 0,
|
||
):
|
||
self.name = name
|
||
self.metric_func = metric_func
|
||
self.threshold = threshold
|
||
self.severity = severity
|
||
self.duration_minutes = duration_minutes
|
||
self.triggered_at: Optional[datetime] = None
|
||
|
||
|
||
class AlertManager:
|
||
"""Manages alert rules and notifications."""
|
||
|
||
def __init__(
|
||
self,
|
||
session: Session,
|
||
webhook_url: Optional[str] = None,
|
||
openclaw_client=None,
|
||
):
|
||
self.session = session
|
||
self.webhook_url = webhook_url
|
||
self.openclaw_client = openclaw_client
|
||
self.rules = [
|
||
AlertRule(
|
||
name="pool_utilization",
|
||
metric_func=calculate_pool_utilization,
|
||
threshold=0.9,
|
||
severity="warning",
|
||
duration_minutes=10,
|
||
),
|
||
AlertRule(
|
||
name="task_backlog",
|
||
metric_func=calculate_task_backlog,
|
||
threshold=50,
|
||
severity="warning",
|
||
duration_minutes=0,
|
||
),
|
||
AlertRule(
|
||
name="stuck_rate",
|
||
metric_func=calculate_stuck_rate,
|
||
threshold=0.1,
|
||
severity="critical",
|
||
duration_minutes=0,
|
||
),
|
||
]
|
||
|
||
def check_rules(self) -> list[AlertHistoryDB]:
|
||
"""Check all alert rules and create alerts if triggered.
|
||
|
||
Returns:
|
||
List of newly created alerts
|
||
"""
|
||
alerts = []
|
||
|
||
for rule in self.rules:
|
||
metric_value = rule.metric_func(self.session)
|
||
|
||
# Check if threshold exceeded
|
||
if metric_value > rule.threshold:
|
||
# Check duration requirement
|
||
if rule.duration_minutes > 0:
|
||
if rule.triggered_at is None:
|
||
rule.triggered_at = utc_now()
|
||
continue
|
||
|
||
duration = (utc_now() - rule.triggered_at).total_seconds() / 60
|
||
if duration < rule.duration_minutes:
|
||
continue
|
||
else:
|
||
rule.triggered_at = utc_now()
|
||
|
||
# Create alert
|
||
alert = self._create_alert(rule, metric_value)
|
||
alerts.append(alert)
|
||
|
||
# Send webhook notification
|
||
if self.webhook_url:
|
||
self._send_webhook(alert)
|
||
# Note: alert→auto-scale link is invoked from the async router
|
||
# via ``AlertManager.maybe_autoscale`` so we stay out of nested
|
||
# event-loop territory.
|
||
else:
|
||
# Reset trigger time
|
||
rule.triggered_at = None
|
||
|
||
return alerts
|
||
|
||
def _create_alert(self, rule: AlertRule, metric_value: float) -> AlertHistoryDB:
|
||
"""Create an alert history record."""
|
||
message = f"{rule.name}: {metric_value:.2f} exceeds threshold {rule.threshold}"
|
||
|
||
alert = AlertHistoryDB(
|
||
id=f"alert-{utc_now().timestamp()}",
|
||
alert_type=rule.name,
|
||
severity=rule.severity,
|
||
message=message,
|
||
metric_value=metric_value,
|
||
threshold=rule.threshold,
|
||
)
|
||
|
||
self.session.add(alert)
|
||
self.session.commit()
|
||
self.session.refresh(alert)
|
||
|
||
_logger.warning(f"Alert triggered: {message}")
|
||
|
||
return alert
|
||
|
||
def _send_webhook(self, alert: AlertHistoryDB, max_retries: int = 3) -> None:
|
||
"""Send webhook with retry and dedupe.
|
||
|
||
P1 真问题修复（§6.2）:
|
||
- **Dedupe**: if ``alert.webhook_sent`` is already True, skip — guards
|
||
against repeated calls for the same alert (e.g. consecutive
|
||
``check_alerts`` ticks).
|
||
- **Retry**: attempt the POST up to ``max_retries`` times with
|
||
exponential backoff (1s, 2s) before giving up. Persist
|
||
``webhook_sent=True`` only on success.
|
||
"""
|
||
if not self.webhook_url:
|
||
return
|
||
if alert.webhook_sent:
|
||
return
|
||
|
||
payload = {
|
||
"alert_id": alert.id,
|
||
"alert_type": alert.alert_type,
|
||
"severity": alert.severity,
|
||
"message": alert.message,
|
||
"metric_value": alert.metric_value,
|
||
"threshold": alert.threshold,
|
||
"timestamp": alert.created_at.isoformat(),
|
||
}
|
||
|
||
import time
|
||
|
||
last_err: Exception | None = None
|
||
for attempt in range(max_retries):
|
||
try:
|
||
response = httpx.post(self.webhook_url, json=payload, timeout=5.0)
|
||
response.raise_for_status()
|
||
alert.webhook_sent = True
|
||
self.session.commit()
|
||
_logger.info(f"Webhook sent for alert {alert.id} (attempt {attempt + 1})")
|
||
return
|
||
except Exception as e:
|
||
last_err = e
|
||
if attempt < max_retries - 1:
|
||
time.sleep(2 ** attempt)
|
||
|
||
_logger.error(f"Failed to send webhook for alert {alert.id} after {max_retries} attempts: {last_err}")
|
||
|
||
def get_alert_history(self, limit: int = 100) -> list[AlertHistoryDB]:
|
||
"""Get alert history.
|
||
|
||
Returns:
|
||
List of alerts, ordered by created_at descending
|
||
"""
|
||
alerts = self.session.exec(
|
||
select(AlertHistoryDB)
|
||
.order_by(AlertHistoryDB.created_at.desc())
|
||
.limit(limit)
|
||
).all()
|
||
return list(alerts)
|
||
|
||
def get_unresolved_alerts(self) -> list[AlertHistoryDB]:
|
||
"""Get unresolved alerts.
|
||
|
||
Returns:
|
||
List of unresolved alerts
|
||
"""
|
||
alerts = self.session.exec(
|
||
select(AlertHistoryDB).where(AlertHistoryDB.resolved_at.is_(None))
|
||
).all()
|
||
return list(alerts)
|
||
|
||
def resolve_alert(self, alert_id: str) -> bool:
|
||
"""Resolve an alert.
|
||
|
||
Returns:
|
||
True if alert was resolved, False if not found
|
||
"""
|
||
alert = self.session.get(AlertHistoryDB, alert_id)
|
||
if alert is None:
|
||
return False
|
||
|
||
alert.resolved_at = utc_now()
|
||
self.session.commit()
|
||
return True
|
||
|
||
|
||
async def maybe_autoscale(self, alert: AlertHistoryDB) -> None:
|
||
"""Async entry point for the alert→auto-scale link (T7).
|
||
|
||
Called by the router after ``check_rules``. Best-effort: failures are
|
||
logged, not raised. ``cron_pool.scale_up`` itself caps at
|
||
``MAX_POOL_SIZE``, so repeated invocations are safe.
|
||
"""
|
||
if self.openclaw_client is None:
|
||
return
|
||
try:
|
||
from agenteval.intelligent_eval import cron_pool as _cp
|
||
await _cp.scale_up(1, self.session, self.openclaw_client)
|
||
except Exception as e:
|
||
_logger.error(f"Auto-scale-up on alert {alert.id} failed: {e}")
|