"""Unit tests for alert rules and notifications.""" from datetime import timedelta from unittest.mock import MagicMock, patch import pytest from sqlmodel import Session, select from agenteval.intelligent_eval.alerts import AlertHistoryDB, AlertManager from agenteval.storage.db import ( IntelligentEvalTaskQueueDB, OpenClawCronPoolDB, utc_now, ) def test_alert_manager_check_rules_no_alerts(db_session: Session): """Test alert manager when no rules are triggered.""" # Create healthy state: low utilization, low backlog, no stuck for i in range(5): cron = OpenClawCronPoolDB( openclaw_cron_id=f"cron-{i}", status="idle", last_active_at=utc_now(), ) db_session.add(cron) db_session.commit() manager = AlertManager(db_session) alerts = manager.check_rules() assert len(alerts) == 0 def test_alert_manager_task_backlog_alert(db_session: Session): """Test alert manager triggers task backlog alert.""" # Create high backlog: 60 pending tasks for i in range(60): task = IntelligentEvalTaskQueueDB( eval_id=f"eval-{i}", status="pending", priority=1, reason="slot_due", ) db_session.add(task) db_session.commit() manager = AlertManager(db_session) alerts = manager.check_rules() # Should trigger task_backlog alert backlog_alerts = [a for a in alerts if a.alert_type == "task_backlog"] assert len(backlog_alerts) == 1 assert backlog_alerts[0].metric_value == 60 assert backlog_alerts[0].threshold == 50 assert backlog_alerts[0].severity == "warning" def test_alert_manager_stuck_rate_alert(db_session: Session): """Test alert manager triggers stuck rate alert.""" # Create high stuck rate: 3 stuck out of 10 for i in range(3): cron = OpenClawCronPoolDB( openclaw_cron_id=f"stuck-{i}", status="stuck", last_active_at=utc_now(), ) db_session.add(cron) for i in range(7): cron = OpenClawCronPoolDB( openclaw_cron_id=f"active-{i}", status="busy", last_active_at=utc_now(), ) db_session.add(cron) db_session.commit() manager = AlertManager(db_session) alerts = manager.check_rules() # Should trigger stuck_rate alert stuck_alerts = [a for a in alerts if a.alert_type == "stuck_rate"] assert len(stuck_alerts) == 1 assert stuck_alerts[0].metric_value == 0.3 assert stuck_alerts[0].threshold == 0.1 assert stuck_alerts[0].severity == "critical" def test_alert_manager_pool_utilization_with_duration(db_session: Session): """Test alert manager respects duration requirement for pool utilization.""" # Create high utilization: 19 busy out of 20 for i in range(19): cron = OpenClawCronPoolDB( openclaw_cron_id=f"busy-{i}", status="busy", last_active_at=utc_now(), ) db_session.add(cron) cron = OpenClawCronPoolDB( openclaw_cron_id="idle-0", status="idle", last_active_at=utc_now(), ) db_session.add(cron) db_session.commit() manager = AlertManager(db_session) # First check: should not trigger (duration not met) alerts1 = manager.check_rules() utilization_alerts1 = [a for a in alerts1 if a.alert_type == "pool_utilization"] assert len(utilization_alerts1) == 0 # Simulate time passing (10 minutes) # In real scenario, this would be checked over time # For testing, we just verify the logic exists def test_alert_manager_get_alert_history(db_session: Session): """Test getting alert history.""" # Create some alerts for i in range(5): alert = AlertHistoryDB( id=f"alert-{i}", alert_type="task_backlog", severity="warning", message=f"Test alert {i}", metric_value=50 + i, threshold=50, ) db_session.add(alert) db_session.commit() manager = AlertManager(db_session) alerts = manager.get_alert_history(limit=3) assert len(alerts) == 3 # Should be ordered by created_at descending assert alerts[0].id == "alert-4" def test_alert_manager_get_unresolved_alerts(db_session: Session): """Test getting unresolved alerts.""" # Create resolved and unresolved alerts alert1 = AlertHistoryDB( id="alert-1", alert_type="task_backlog", severity="warning", message="Test alert 1", metric_value=60, threshold=50, ) alert2 = AlertHistoryDB( id="alert-2", alert_type="stuck_rate", severity="critical", message="Test alert 2", metric_value=0.2, threshold=0.1, resolved_at=utc_now(), ) alert3 = AlertHistoryDB( id="alert-3", alert_type="pool_utilization", severity="warning", message="Test alert 3", metric_value=0.95, threshold=0.9, ) db_session.add_all([alert1, alert2, alert3]) db_session.commit() manager = AlertManager(db_session) unresolved = manager.get_unresolved_alerts() assert len(unresolved) == 2 assert all(a.resolved_at is None for a in unresolved) def test_alert_manager_resolve_alert(db_session: Session): """Test resolving an alert.""" alert = AlertHistoryDB( id="alert-1", alert_type="task_backlog", severity="warning", message="Test alert", metric_value=60, threshold=50, ) db_session.add(alert) db_session.commit() manager = AlertManager(db_session) resolved = manager.resolve_alert("alert-1") assert resolved is True db_session.refresh(alert) assert alert.resolved_at is not None def test_alert_manager_resolve_nonexistent_alert(db_session: Session): """Test resolving a non-existent alert.""" manager = AlertManager(db_session) resolved = manager.resolve_alert("nonexistent") assert resolved is False def test_alert_manager_webhook_notification(db_session: Session): """Test webhook notification.""" # Create high backlog to trigger alert for i in range(60): task = IntelligentEvalTaskQueueDB( eval_id=f"eval-{i}", status="pending", priority=1, reason="slot_due", ) db_session.add(task) db_session.commit() # Mock webhook with patch("httpx.post") as mock_post: mock_post.return_value.status_code = 200 mock_post.return_value.raise_for_status = MagicMock() manager = AlertManager(db_session, webhook_url="https://example.com/webhook") alerts = manager.check_rules() # Should have triggered alerts assert len(alerts) > 0 # Should have called webhook assert mock_post.called def test_alert_manager_webhook_failure(db_session: Session): """Test webhook notification failure.""" # Create high backlog to trigger alert for i in range(60): task = IntelligentEvalTaskQueueDB( eval_id=f"eval-{i}", status="pending", priority=1, reason="slot_due", ) db_session.add(task) db_session.commit() # Mock webhook failure with patch("httpx.post") as mock_post: mock_post.side_effect = Exception("Webhook failed") manager = AlertManager(db_session, webhook_url="https://example.com/webhook") alerts = manager.check_rules() # Should still create alerts even if webhook fails assert len(alerts) > 0 # Webhook should not be marked as sent for alert in alerts: assert alert.webhook_sent is False