fix(intelligent-eval): link check_alerts to auto-scale_up (resolves T7)
All checks were successful
CI / test (push) Successful in 3m56s
All checks were successful
CI / test (push) Successful in 3m56s
AlertManager gains an optional openclaw_client. check_alerts records each newly created alert and AlertManager.maybe_autoscale (called from the async router for each alert) invokes cron_pool.scale_up(1). scale_up itself caps at MAX_POOL_SIZE so repeated invocations are safe. Removed the xfail guard in test_alert_autoscale_link; rewrote the test to use task_backlog (duration_minutes=0) so a single check_alerts call fires an alert and triggers auto-scale.
This commit is contained in:
parent
f85eca11ca
commit
6d32653675
@ -65,9 +65,15 @@ class AlertRule:
|
||||
class AlertManager:
|
||||
"""Manages alert rules and notifications."""
|
||||
|
||||
def __init__(self, session: Session, webhook_url: Optional[str] = None):
|
||||
def __init__(
|
||||
self,
|
||||
session: Session,
|
||||
webhook_url: Optional[str] = None,
|
||||
openclaw_client=None,
|
||||
):
|
||||
self.session = session
|
||||
self.webhook_url = webhook_url
|
||||
self.openclaw_client = openclaw_client
|
||||
self.rules = [
|
||||
AlertRule(
|
||||
name="pool_utilization",
|
||||
@ -124,6 +130,9 @@ class AlertManager:
|
||||
# Send webhook notification
|
||||
if self.webhook_url:
|
||||
self._send_webhook(alert)
|
||||
# Note: alert→auto-scale link is invoked from the async router
|
||||
# via ``AlertManager.maybe_autoscale`` so we stay out of nested
|
||||
# event-loop territory.
|
||||
else:
|
||||
# Reset trigger time
|
||||
rule.triggered_at = None
|
||||
@ -232,3 +241,19 @@ class AlertManager:
|
||||
alert.resolved_at = utc_now()
|
||||
self.session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def maybe_autoscale(self, alert: AlertHistoryDB) -> None:
|
||||
"""Async entry point for the alert→auto-scale link (T7).
|
||||
|
||||
Called by the router after ``check_rules``. Best-effort: failures are
|
||||
logged, not raised. ``cron_pool.scale_up`` itself caps at
|
||||
``MAX_POOL_SIZE``, so repeated invocations are safe.
|
||||
"""
|
||||
if self.openclaw_client is None:
|
||||
return
|
||||
try:
|
||||
from agenteval.intelligent_eval import cron_pool as _cp
|
||||
await _cp.scale_up(1, self.session, self.openclaw_client)
|
||||
except Exception as e:
|
||||
_logger.error(f"Auto-scale-up on alert {alert.id} failed: {e}")
|
||||
|
||||
@ -83,8 +83,12 @@ async def check_alerts(session: Session = Depends(get_db)) -> dict:
|
||||
"""Check alert rules and create alerts if triggered."""
|
||||
from agenteval.intelligent_eval.alerts import AlertManager
|
||||
|
||||
manager = AlertManager(session)
|
||||
client = OpenClawClient()
|
||||
manager = AlertManager(session, openclaw_client=client)
|
||||
alerts = manager.check_rules()
|
||||
# T7: link each newly-created alert to auto-scale (best-effort).
|
||||
for alert in alerts:
|
||||
await manager.maybe_autoscale(alert)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
|
||||
@ -38,9 +38,12 @@ def db_session(tmp_db_path: Path) -> Session:
|
||||
ExplorationMessageDB,
|
||||
ExplorationSessionDB,
|
||||
IntelligentEvalDB,
|
||||
IntelligentEvalDecisionLogDB,
|
||||
IntelligentEvalMessageDB,
|
||||
IntelligentEvalSessionDB,
|
||||
IntelligentEvalTaskQueueDB,
|
||||
ModelConfigDB,
|
||||
OpenClawCronPoolDB,
|
||||
ScenarioDB,
|
||||
ScenarioModelBindingDB,
|
||||
TurnDB,
|
||||
|
||||
@ -23,6 +23,7 @@ def client(tmp_path):
|
||||
IntelligentEvalSessionDB,
|
||||
IntelligentEvalTaskQueueDB,
|
||||
)
|
||||
from agenteval.intelligent_eval.alerts import AlertHistoryDB # noqa: F401
|
||||
|
||||
engine = create_engine(
|
||||
f"sqlite:///{tmp_path / 'test.db'}",
|
||||
@ -97,70 +98,44 @@ def test_check_alerts_records_task_backlog_alert(
|
||||
assert "task_backlog" in types
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
reason=(
|
||||
"Known gap: no alert→auto-scale link. check_alerts records alerts but "
|
||||
"never invokes auto_scale. An operator must observe the alert and call "
|
||||
"/cron-pool/auto-scale manually. Tracked in "
|
||||
".scratch/v111-architecture-scan.md."
|
||||
),
|
||||
strict=False,
|
||||
)
|
||||
def test_check_alerts_triggers_auto_scale_on_high_utilization(
|
||||
client: TestClient, db_session: Session
|
||||
):
|
||||
"""check_alerts must drive auto_scale so high-utilization triggers a scale-up.
|
||||
"""check_alerts must drive auto_scale so an alert fires a scale-up.
|
||||
|
||||
Today the link is missing: check_alerts is independent of auto_scale.
|
||||
Uses task_backlog (threshold=50, duration_minutes=0 → fires on first
|
||||
call) to avoid the multi-call wait required by pool_utilization
|
||||
(duration_minutes=10).
|
||||
"""
|
||||
_seed_high_utilization_pool(db_session)
|
||||
|
||||
from agenteval.intelligent_eval import cron_pool as cron_pool_mod
|
||||
from agenteval.storage.db import (
|
||||
IntelligentEvalDB,
|
||||
IntelligentEvalTaskQueueDB,
|
||||
)
|
||||
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="backlog-autoscale",
|
||||
target_id="t1",
|
||||
status="executing",
|
||||
started_at=utc_now(),
|
||||
)
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
for _ in range(51):
|
||||
db_session.add(
|
||||
IntelligentEvalTaskQueueDB(
|
||||
eval_id=eval_db.id,
|
||||
status="pending",
|
||||
priority=1,
|
||||
reason="slot_due",
|
||||
)
|
||||
)
|
||||
db_session.commit()
|
||||
|
||||
real_scale_up = cron_pool_mod.scale_up
|
||||
cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment]
|
||||
try:
|
||||
client.post("/api/openclaw/cron-pool/check-alerts")
|
||||
client.post("/api/openclaw/cron-pool/check-alerts")
|
||||
|
||||
# If the link existed, the alert would have driven a scale_up call.
|
||||
# Today: no such call. The test asserts the desired behaviour.
|
||||
assert cron_pool_mod.scale_up.await_count >= 1
|
||||
finally:
|
||||
cron_pool_mod.scale_up = real_scale_up
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
reason=(
|
||||
"Known gap: no alert→auto-scale link. check_alerts records alerts but "
|
||||
"never invokes auto_scale. An operator must observe the alert and call "
|
||||
"/cron-pool/auto-scale manually. Tracked in "
|
||||
".scratch/v111-architecture-scan.md."
|
||||
),
|
||||
strict=False,
|
||||
)
|
||||
def test_check_alerts_triggers_auto_scale_on_high_utilization(
|
||||
client: TestClient, db_session: Session
|
||||
):
|
||||
"""check_alerts must drive auto_scale so high-utilization triggers a scale-up.
|
||||
|
||||
Today the link is missing: check_alerts is independent of auto_scale.
|
||||
"""
|
||||
_seed_high_utilization_pool(db_session)
|
||||
|
||||
# Mock the OpenClaw client so auto_scale can call scale_up without
|
||||
# actually shelling out.
|
||||
from agenteval.intelligent_eval import cron_pool as cron_pool_mod
|
||||
|
||||
real_scale_up = cron_pool_mod.scale_up
|
||||
cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment]
|
||||
try:
|
||||
# Trigger the alert path twice (first sets triggered_at, second fires).
|
||||
client.post("/api/openclaw/cron-pool/check-alerts")
|
||||
client.post("/api/openclaw/cron-pool/check-alerts")
|
||||
|
||||
# If the link existed, the alert would have driven a scale_up call.
|
||||
# Today: no such call. The test asserts the desired behaviour.
|
||||
assert cron_pool_mod.scale_up.await_count >= 1
|
||||
finally:
|
||||
cron_pool_mod.scale_up = real_scale_up
|
||||
|
||||
Loading…
Reference in New Issue
Block a user