fix(alerts): webhook retry + dedupe (resolves §6.2)
All checks were successful
CI / test (push) Successful in 3m57s

_send_webhook:
- Dedupe: skip if alert.webhook_sent is already True (guards against
  repeated check_alerts ticks re-sending the same alert).
- Retry: up to 3 attempts with exponential backoff (1s, 2s) before
  giving up. webhook_sent=True is persisted only on success.

Two xfail guards in test_openclaw_client_and_webhook now pass (876/4 xfail).
This commit is contained in:
sinohqb 2026-08-14 14:54:57 +08:00
parent 38e3817433
commit f85eca11ca
2 changed files with 37 additions and 34 deletions

View File

@ -151,12 +151,22 @@ class AlertManager:
return alert return alert
def _send_webhook(self, alert: AlertHistoryDB) -> None: def _send_webhook(self, alert: AlertHistoryDB, max_retries: int = 3) -> None:
"""Send webhook notification.""" """Send webhook with retry and dedupe.
P1 真问题修复§6.2:
- **Dedupe**: if ``alert.webhook_sent`` is already True, skip guards
against repeated calls for the same alert (e.g. consecutive
``check_alerts`` ticks).
- **Retry**: attempt the POST up to ``max_retries`` times with
exponential backoff (1s, 2s) before giving up. Persist
``webhook_sent=True`` only on success.
"""
if not self.webhook_url: if not self.webhook_url:
return return
if alert.webhook_sent:
return
try:
payload = { payload = {
"alert_id": alert.id, "alert_id": alert.id,
"alert_type": alert.alert_type, "alert_type": alert.alert_type,
@ -167,15 +177,23 @@ class AlertManager:
"timestamp": alert.created_at.isoformat(), "timestamp": alert.created_at.isoformat(),
} }
import time
last_err: Exception | None = None
for attempt in range(max_retries):
try:
response = httpx.post(self.webhook_url, json=payload, timeout=5.0) response = httpx.post(self.webhook_url, json=payload, timeout=5.0)
response.raise_for_status() response.raise_for_status()
alert.webhook_sent = True alert.webhook_sent = True
self.session.commit() self.session.commit()
_logger.info(f"Webhook sent for alert {alert.id} (attempt {attempt + 1})")
_logger.info(f"Webhook sent for alert {alert.id}") return
except Exception as e: except Exception as e:
_logger.error(f"Failed to send webhook: {e}") last_err = e
if attempt < max_retries - 1:
time.sleep(2 ** attempt)
_logger.error(f"Failed to send webhook for alert {alert.id} after {max_retries} attempts: {last_err}")
def get_alert_history(self, limit: int = 100) -> list[AlertHistoryDB]: def get_alert_history(self, limit: int = 100) -> list[AlertHistoryDB]:
"""Get alert history. """Get alert history.

View File

@ -139,14 +139,6 @@ def test_webhook_failure_does_not_raise_or_mark_sent(db_session: Session):
assert alert.webhook_sent is False assert alert.webhook_sent is False
@pytest.mark.xfail(
reason=(
"Known gap: webhook has no retry. _send_webhook swallows the first "
"failure and leaves webhook_sent=False; no follow-up attempt is made. "
"Tracked in .scratch/v111-architecture-scan.md."
),
strict=False,
)
def test_webhook_retries_on_failure(db_session: Session): def test_webhook_retries_on_failure(db_session: Session):
alert = _make_alert(db_session) alert = _make_alert(db_session)
fake_resp = MagicMock(); fake_resp.raise_for_status = MagicMock() fake_resp = MagicMock(); fake_resp.raise_for_status = MagicMock()
@ -160,13 +152,6 @@ def test_webhook_retries_on_failure(db_session: Session):
assert alert.webhook_sent is True assert alert.webhook_sent is True
@pytest.mark.xfail(
reason=(
"Known gap: no dedupe — every check_alerts trigger re-sends webhook "
"for the same alert. Tracked in .scratch/v111-architecture-scan.md."
),
strict=False,
)
def test_webhook_dedupes_repeat_triggers(db_session: Session): def test_webhook_dedupes_repeat_triggers(db_session: Session):
"""Calling _send_webhook twice on the same alert must POST at most once.""" """Calling _send_webhook twice on the same alert must POST at most once."""
alert = _make_alert(db_session) alert = _make_alert(db_session)