fix(alerts): webhook retry + dedupe (resolves §6.2)
All checks were successful
CI / test (push) Successful in 3m57s
All checks were successful
CI / test (push) Successful in 3m57s
_send_webhook: - Dedupe: skip if alert.webhook_sent is already True (guards against repeated check_alerts ticks re-sending the same alert). - Retry: up to 3 attempts with exponential backoff (1s, 2s) before giving up. webhook_sent=True is persisted only on success. Two xfail guards in test_openclaw_client_and_webhook now pass (876/4 xfail).
This commit is contained in:
parent
38e3817433
commit
f85eca11ca
@ -151,31 +151,49 @@ class AlertManager:
|
|||||||
|
|
||||||
return alert
|
return alert
|
||||||
|
|
||||||
def _send_webhook(self, alert: AlertHistoryDB) -> None:
|
def _send_webhook(self, alert: AlertHistoryDB, max_retries: int = 3) -> None:
|
||||||
"""Send webhook notification."""
|
"""Send webhook with retry and dedupe.
|
||||||
|
|
||||||
|
P1 真问题修复(§6.2):
|
||||||
|
- **Dedupe**: if ``alert.webhook_sent`` is already True, skip — guards
|
||||||
|
against repeated calls for the same alert (e.g. consecutive
|
||||||
|
``check_alerts`` ticks).
|
||||||
|
- **Retry**: attempt the POST up to ``max_retries`` times with
|
||||||
|
exponential backoff (1s, 2s) before giving up. Persist
|
||||||
|
``webhook_sent=True`` only on success.
|
||||||
|
"""
|
||||||
if not self.webhook_url:
|
if not self.webhook_url:
|
||||||
return
|
return
|
||||||
|
if alert.webhook_sent:
|
||||||
|
return
|
||||||
|
|
||||||
try:
|
payload = {
|
||||||
payload = {
|
"alert_id": alert.id,
|
||||||
"alert_id": alert.id,
|
"alert_type": alert.alert_type,
|
||||||
"alert_type": alert.alert_type,
|
"severity": alert.severity,
|
||||||
"severity": alert.severity,
|
"message": alert.message,
|
||||||
"message": alert.message,
|
"metric_value": alert.metric_value,
|
||||||
"metric_value": alert.metric_value,
|
"threshold": alert.threshold,
|
||||||
"threshold": alert.threshold,
|
"timestamp": alert.created_at.isoformat(),
|
||||||
"timestamp": alert.created_at.isoformat(),
|
}
|
||||||
}
|
|
||||||
|
|
||||||
response = httpx.post(self.webhook_url, json=payload, timeout=5.0)
|
import time
|
||||||
response.raise_for_status()
|
|
||||||
|
|
||||||
alert.webhook_sent = True
|
last_err: Exception | None = None
|
||||||
self.session.commit()
|
for attempt in range(max_retries):
|
||||||
|
try:
|
||||||
|
response = httpx.post(self.webhook_url, json=payload, timeout=5.0)
|
||||||
|
response.raise_for_status()
|
||||||
|
alert.webhook_sent = True
|
||||||
|
self.session.commit()
|
||||||
|
_logger.info(f"Webhook sent for alert {alert.id} (attempt {attempt + 1})")
|
||||||
|
return
|
||||||
|
except Exception as e:
|
||||||
|
last_err = e
|
||||||
|
if attempt < max_retries - 1:
|
||||||
|
time.sleep(2 ** attempt)
|
||||||
|
|
||||||
_logger.info(f"Webhook sent for alert {alert.id}")
|
_logger.error(f"Failed to send webhook for alert {alert.id} after {max_retries} attempts: {last_err}")
|
||||||
except Exception as e:
|
|
||||||
_logger.error(f"Failed to send webhook: {e}")
|
|
||||||
|
|
||||||
def get_alert_history(self, limit: int = 100) -> list[AlertHistoryDB]:
|
def get_alert_history(self, limit: int = 100) -> list[AlertHistoryDB]:
|
||||||
"""Get alert history.
|
"""Get alert history.
|
||||||
|
|||||||
@ -139,14 +139,6 @@ def test_webhook_failure_does_not_raise_or_mark_sent(db_session: Session):
|
|||||||
assert alert.webhook_sent is False
|
assert alert.webhook_sent is False
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.xfail(
|
|
||||||
reason=(
|
|
||||||
"Known gap: webhook has no retry. _send_webhook swallows the first "
|
|
||||||
"failure and leaves webhook_sent=False; no follow-up attempt is made. "
|
|
||||||
"Tracked in .scratch/v111-architecture-scan.md."
|
|
||||||
),
|
|
||||||
strict=False,
|
|
||||||
)
|
|
||||||
def test_webhook_retries_on_failure(db_session: Session):
|
def test_webhook_retries_on_failure(db_session: Session):
|
||||||
alert = _make_alert(db_session)
|
alert = _make_alert(db_session)
|
||||||
fake_resp = MagicMock(); fake_resp.raise_for_status = MagicMock()
|
fake_resp = MagicMock(); fake_resp.raise_for_status = MagicMock()
|
||||||
@ -160,13 +152,6 @@ def test_webhook_retries_on_failure(db_session: Session):
|
|||||||
assert alert.webhook_sent is True
|
assert alert.webhook_sent is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.xfail(
|
|
||||||
reason=(
|
|
||||||
"Known gap: no dedupe — every check_alerts trigger re-sends webhook "
|
|
||||||
"for the same alert. Tracked in .scratch/v111-architecture-scan.md."
|
|
||||||
),
|
|
||||||
strict=False,
|
|
||||||
)
|
|
||||||
def test_webhook_dedupes_repeat_triggers(db_session: Session):
|
def test_webhook_dedupes_repeat_triggers(db_session: Session):
|
||||||
"""Calling _send_webhook twice on the same alert must POST at most once."""
|
"""Calling _send_webhook twice on the same alert must POST at most once."""
|
||||||
alert = _make_alert(db_session)
|
alert = _make_alert(db_session)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user