fix(intelligent-eval): link check_alerts to auto-scale_up (resolves T7)
All checks were successful
CI / test (push) Successful in 3m56s

AlertManager gains an optional openclaw_client. check_alerts records
each newly created alert and AlertManager.maybe_autoscale (called
from the async router for each alert) invokes cron_pool.scale_up(1).
scale_up itself caps at MAX_POOL_SIZE so repeated invocations are safe.

Removed the xfail guard in test_alert_autoscale_link; rewrote the test
to use task_backlog (duration_minutes=0) so a single check_alerts call
fires an alert and triggers auto-scale.
This commit is contained in:
sinohqb 2026-08-14 15:08:24 +08:00
parent f85eca11ca
commit 6d32653675
4 changed files with 62 additions and 55 deletions

View File

@ -65,9 +65,15 @@ class AlertRule:
class AlertManager:
"""Manages alert rules and notifications."""
def __init__(self, session: Session, webhook_url: Optional[str] = None):
def __init__(
self,
session: Session,
webhook_url: Optional[str] = None,
openclaw_client=None,
):
self.session = session
self.webhook_url = webhook_url
self.openclaw_client = openclaw_client
self.rules = [
AlertRule(
name="pool_utilization",
@ -124,6 +130,9 @@ class AlertManager:
# Send webhook notification
if self.webhook_url:
self._send_webhook(alert)
# Note: alert→auto-scale link is invoked from the async router
# via ``AlertManager.maybe_autoscale`` so we stay out of nested
# event-loop territory.
else:
# Reset trigger time
rule.triggered_at = None
@ -232,3 +241,19 @@ class AlertManager:
alert.resolved_at = utc_now()
self.session.commit()
return True
async def maybe_autoscale(self, alert: AlertHistoryDB) -> None:
"""Async entry point for the alert→auto-scale link (T7).
Called by the router after ``check_rules``. Best-effort: failures are
logged, not raised. ``cron_pool.scale_up`` itself caps at
``MAX_POOL_SIZE``, so repeated invocations are safe.
"""
if self.openclaw_client is None:
return
try:
from agenteval.intelligent_eval import cron_pool as _cp
await _cp.scale_up(1, self.session, self.openclaw_client)
except Exception as e:
_logger.error(f"Auto-scale-up on alert {alert.id} failed: {e}")

View File

@ -83,8 +83,12 @@ async def check_alerts(session: Session = Depends(get_db)) -> dict:
"""Check alert rules and create alerts if triggered."""
from agenteval.intelligent_eval.alerts import AlertManager
manager = AlertManager(session)
client = OpenClawClient()
manager = AlertManager(session, openclaw_client=client)
alerts = manager.check_rules()
# T7: link each newly-created alert to auto-scale (best-effort).
for alert in alerts:
await manager.maybe_autoscale(alert)
return {
"success": True,

View File

@ -38,9 +38,12 @@ def db_session(tmp_db_path: Path) -> Session:
ExplorationMessageDB,
ExplorationSessionDB,
IntelligentEvalDB,
IntelligentEvalDecisionLogDB,
IntelligentEvalMessageDB,
IntelligentEvalSessionDB,
IntelligentEvalTaskQueueDB,
ModelConfigDB,
OpenClawCronPoolDB,
ScenarioDB,
ScenarioModelBindingDB,
TurnDB,

View File

@ -23,6 +23,7 @@ def client(tmp_path):
IntelligentEvalSessionDB,
IntelligentEvalTaskQueueDB,
)
from agenteval.intelligent_eval.alerts import AlertHistoryDB # noqa: F401
engine = create_engine(
f"sqlite:///{tmp_path / 'test.db'}",
@ -97,70 +98,44 @@ def test_check_alerts_records_task_backlog_alert(
assert "task_backlog" in types
@pytest.mark.xfail(
reason=(
"Known gap: no alert→auto-scale link. check_alerts records alerts but "
"never invokes auto_scale. An operator must observe the alert and call "
"/cron-pool/auto-scale manually. Tracked in "
".scratch/v111-architecture-scan.md."
),
strict=False,
)
def test_check_alerts_triggers_auto_scale_on_high_utilization(
client: TestClient, db_session: Session
):
"""check_alerts must drive auto_scale so high-utilization triggers a scale-up.
"""check_alerts must drive auto_scale so an alert fires a scale-up.
Today the link is missing: check_alerts is independent of auto_scale.
Uses task_backlog (threshold=50, duration_minutes=0 fires on first
call) to avoid the multi-call wait required by pool_utilization
(duration_minutes=10).
"""
_seed_high_utilization_pool(db_session)
from agenteval.intelligent_eval import cron_pool as cron_pool_mod
from agenteval.storage.db import (
IntelligentEvalDB,
IntelligentEvalTaskQueueDB,
)
eval_db = IntelligentEvalDB(
name="backlog-autoscale",
target_id="t1",
status="executing",
started_at=utc_now(),
)
db_session.add(eval_db)
db_session.commit()
for _ in range(51):
db_session.add(
IntelligentEvalTaskQueueDB(
eval_id=eval_db.id,
status="pending",
priority=1,
reason="slot_due",
)
)
db_session.commit()
real_scale_up = cron_pool_mod.scale_up
cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment]
try:
client.post("/api/openclaw/cron-pool/check-alerts")
client.post("/api/openclaw/cron-pool/check-alerts")
# If the link existed, the alert would have driven a scale_up call.
# Today: no such call. The test asserts the desired behaviour.
assert cron_pool_mod.scale_up.await_count >= 1
finally:
cron_pool_mod.scale_up = real_scale_up
@pytest.mark.xfail(
reason=(
"Known gap: no alert→auto-scale link. check_alerts records alerts but "
"never invokes auto_scale. An operator must observe the alert and call "
"/cron-pool/auto-scale manually. Tracked in "
".scratch/v111-architecture-scan.md."
),
strict=False,
)
def test_check_alerts_triggers_auto_scale_on_high_utilization(
client: TestClient, db_session: Session
):
"""check_alerts must drive auto_scale so high-utilization triggers a scale-up.
Today the link is missing: check_alerts is independent of auto_scale.
"""
_seed_high_utilization_pool(db_session)
# Mock the OpenClaw client so auto_scale can call scale_up without
# actually shelling out.
from agenteval.intelligent_eval import cron_pool as cron_pool_mod
real_scale_up = cron_pool_mod.scale_up
cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment]
try:
# Trigger the alert path twice (first sets triggered_at, second fires).
client.post("/api/openclaw/cron-pool/check-alerts")
client.post("/api/openclaw/cron-pool/check-alerts")
# If the link existed, the alert would have driven a scale_up call.
# Today: no such call. The test asserts the desired behaviour.
assert cron_pool_mod.scale_up.await_count >= 1
finally:
cron_pool_mod.scale_up = real_scale_up