diff --git a/backend/agenteval/intelligent_eval/alerts.py b/backend/agenteval/intelligent_eval/alerts.py index 9965633..14b28f8 100644 --- a/backend/agenteval/intelligent_eval/alerts.py +++ b/backend/agenteval/intelligent_eval/alerts.py @@ -65,9 +65,15 @@ class AlertRule: class AlertManager: """Manages alert rules and notifications.""" - def __init__(self, session: Session, webhook_url: Optional[str] = None): + def __init__( + self, + session: Session, + webhook_url: Optional[str] = None, + openclaw_client=None, + ): self.session = session self.webhook_url = webhook_url + self.openclaw_client = openclaw_client self.rules = [ AlertRule( name="pool_utilization", @@ -124,6 +130,9 @@ class AlertManager: # Send webhook notification if self.webhook_url: self._send_webhook(alert) + # Note: alert→auto-scale link is invoked from the async router + # via ``AlertManager.maybe_autoscale`` so we stay out of nested + # event-loop territory. else: # Reset trigger time rule.triggered_at = None @@ -232,3 +241,19 @@ class AlertManager: alert.resolved_at = utc_now() self.session.commit() return True + + + async def maybe_autoscale(self, alert: AlertHistoryDB) -> None: + """Async entry point for the alert→auto-scale link (T7). + + Called by the router after ``check_rules``. Best-effort: failures are + logged, not raised. ``cron_pool.scale_up`` itself caps at + ``MAX_POOL_SIZE``, so repeated invocations are safe. + """ + if self.openclaw_client is None: + return + try: + from agenteval.intelligent_eval import cron_pool as _cp + await _cp.scale_up(1, self.session, self.openclaw_client) + except Exception as e: + _logger.error(f"Auto-scale-up on alert {alert.id} failed: {e}") diff --git a/backend/agenteval/web/routers/openclaw_cron_pool.py b/backend/agenteval/web/routers/openclaw_cron_pool.py index e6da309..8d4258e 100644 --- a/backend/agenteval/web/routers/openclaw_cron_pool.py +++ b/backend/agenteval/web/routers/openclaw_cron_pool.py @@ -83,8 +83,12 @@ async def check_alerts(session: Session = Depends(get_db)) -> dict: """Check alert rules and create alerts if triggered.""" from agenteval.intelligent_eval.alerts import AlertManager - manager = AlertManager(session) + client = OpenClawClient() + manager = AlertManager(session, openclaw_client=client) alerts = manager.check_rules() + # T7: link each newly-created alert to auto-scale (best-effort). + for alert in alerts: + await manager.maybe_autoscale(alert) return { "success": True, diff --git a/tests/conftest.py b/tests/conftest.py index 0dce9cb..e6cf834 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,9 +38,12 @@ def db_session(tmp_db_path: Path) -> Session: ExplorationMessageDB, ExplorationSessionDB, IntelligentEvalDB, + IntelligentEvalDecisionLogDB, IntelligentEvalMessageDB, IntelligentEvalSessionDB, + IntelligentEvalTaskQueueDB, ModelConfigDB, + OpenClawCronPoolDB, ScenarioDB, ScenarioModelBindingDB, TurnDB, diff --git a/tests/integration/test_alert_autoscale_link.py b/tests/integration/test_alert_autoscale_link.py index c1f95e7..2c60439 100644 --- a/tests/integration/test_alert_autoscale_link.py +++ b/tests/integration/test_alert_autoscale_link.py @@ -23,6 +23,7 @@ def client(tmp_path): IntelligentEvalSessionDB, IntelligentEvalTaskQueueDB, ) + from agenteval.intelligent_eval.alerts import AlertHistoryDB # noqa: F401 engine = create_engine( f"sqlite:///{tmp_path / 'test.db'}", @@ -97,70 +98,44 @@ def test_check_alerts_records_task_backlog_alert( assert "task_backlog" in types -@pytest.mark.xfail( - reason=( - "Known gap: no alert→auto-scale link. check_alerts records alerts but " - "never invokes auto_scale. An operator must observe the alert and call " - "/cron-pool/auto-scale manually. Tracked in " - ".scratch/v111-architecture-scan.md." - ), - strict=False, -) def test_check_alerts_triggers_auto_scale_on_high_utilization( client: TestClient, db_session: Session ): - """check_alerts must drive auto_scale so high-utilization triggers a scale-up. + """check_alerts must drive auto_scale so an alert fires a scale-up. - Today the link is missing: check_alerts is independent of auto_scale. + Uses task_backlog (threshold=50, duration_minutes=0 → fires on first + call) to avoid the multi-call wait required by pool_utilization + (duration_minutes=10). """ - _seed_high_utilization_pool(db_session) - from agenteval.intelligent_eval import cron_pool as cron_pool_mod + from agenteval.storage.db import ( + IntelligentEvalDB, + IntelligentEvalTaskQueueDB, + ) + + eval_db = IntelligentEvalDB( + name="backlog-autoscale", + target_id="t1", + status="executing", + started_at=utc_now(), + ) + db_session.add(eval_db) + db_session.commit() + for _ in range(51): + db_session.add( + IntelligentEvalTaskQueueDB( + eval_id=eval_db.id, + status="pending", + priority=1, + reason="slot_due", + ) + ) + db_session.commit() real_scale_up = cron_pool_mod.scale_up cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment] try: client.post("/api/openclaw/cron-pool/check-alerts") - client.post("/api/openclaw/cron-pool/check-alerts") - - # If the link existed, the alert would have driven a scale_up call. - # Today: no such call. The test asserts the desired behaviour. - assert cron_pool_mod.scale_up.await_count >= 1 - finally: - cron_pool_mod.scale_up = real_scale_up - - -@pytest.mark.xfail( - reason=( - "Known gap: no alert→auto-scale link. check_alerts records alerts but " - "never invokes auto_scale. An operator must observe the alert and call " - "/cron-pool/auto-scale manually. Tracked in " - ".scratch/v111-architecture-scan.md." - ), - strict=False, -) -def test_check_alerts_triggers_auto_scale_on_high_utilization( - client: TestClient, db_session: Session -): - """check_alerts must drive auto_scale so high-utilization triggers a scale-up. - - Today the link is missing: check_alerts is independent of auto_scale. - """ - _seed_high_utilization_pool(db_session) - - # Mock the OpenClaw client so auto_scale can call scale_up without - # actually shelling out. - from agenteval.intelligent_eval import cron_pool as cron_pool_mod - - real_scale_up = cron_pool_mod.scale_up - cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment] - try: - # Trigger the alert path twice (first sets triggered_at, second fires). - client.post("/api/openclaw/cron-pool/check-alerts") - client.post("/api/openclaw/cron-pool/check-alerts") - - # If the link existed, the alert would have driven a scale_up call. - # Today: no such call. The test asserts the desired behaviour. assert cron_pool_mod.scale_up.await_count >= 1 finally: cron_pool_mod.scale_up = real_scale_up