fix(intelligent-eval): link check_alerts to auto-scale_up (resolves T7)
All checks were successful
CI / test (push) Successful in 3m56s
All checks were successful
CI / test (push) Successful in 3m56s
AlertManager gains an optional openclaw_client. check_alerts records each newly created alert and AlertManager.maybe_autoscale (called from the async router for each alert) invokes cron_pool.scale_up(1). scale_up itself caps at MAX_POOL_SIZE so repeated invocations are safe. Removed the xfail guard in test_alert_autoscale_link; rewrote the test to use task_backlog (duration_minutes=0) so a single check_alerts call fires an alert and triggers auto-scale.
This commit is contained in:
parent
f85eca11ca
commit
6d32653675
@ -65,9 +65,15 @@ class AlertRule:
|
|||||||
class AlertManager:
|
class AlertManager:
|
||||||
"""Manages alert rules and notifications."""
|
"""Manages alert rules and notifications."""
|
||||||
|
|
||||||
def __init__(self, session: Session, webhook_url: Optional[str] = None):
|
def __init__(
|
||||||
|
self,
|
||||||
|
session: Session,
|
||||||
|
webhook_url: Optional[str] = None,
|
||||||
|
openclaw_client=None,
|
||||||
|
):
|
||||||
self.session = session
|
self.session = session
|
||||||
self.webhook_url = webhook_url
|
self.webhook_url = webhook_url
|
||||||
|
self.openclaw_client = openclaw_client
|
||||||
self.rules = [
|
self.rules = [
|
||||||
AlertRule(
|
AlertRule(
|
||||||
name="pool_utilization",
|
name="pool_utilization",
|
||||||
@ -124,6 +130,9 @@ class AlertManager:
|
|||||||
# Send webhook notification
|
# Send webhook notification
|
||||||
if self.webhook_url:
|
if self.webhook_url:
|
||||||
self._send_webhook(alert)
|
self._send_webhook(alert)
|
||||||
|
# Note: alert→auto-scale link is invoked from the async router
|
||||||
|
# via ``AlertManager.maybe_autoscale`` so we stay out of nested
|
||||||
|
# event-loop territory.
|
||||||
else:
|
else:
|
||||||
# Reset trigger time
|
# Reset trigger time
|
||||||
rule.triggered_at = None
|
rule.triggered_at = None
|
||||||
@ -232,3 +241,19 @@ class AlertManager:
|
|||||||
alert.resolved_at = utc_now()
|
alert.resolved_at = utc_now()
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
async def maybe_autoscale(self, alert: AlertHistoryDB) -> None:
|
||||||
|
"""Async entry point for the alert→auto-scale link (T7).
|
||||||
|
|
||||||
|
Called by the router after ``check_rules``. Best-effort: failures are
|
||||||
|
logged, not raised. ``cron_pool.scale_up`` itself caps at
|
||||||
|
``MAX_POOL_SIZE``, so repeated invocations are safe.
|
||||||
|
"""
|
||||||
|
if self.openclaw_client is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
from agenteval.intelligent_eval import cron_pool as _cp
|
||||||
|
await _cp.scale_up(1, self.session, self.openclaw_client)
|
||||||
|
except Exception as e:
|
||||||
|
_logger.error(f"Auto-scale-up on alert {alert.id} failed: {e}")
|
||||||
|
|||||||
@ -83,8 +83,12 @@ async def check_alerts(session: Session = Depends(get_db)) -> dict:
|
|||||||
"""Check alert rules and create alerts if triggered."""
|
"""Check alert rules and create alerts if triggered."""
|
||||||
from agenteval.intelligent_eval.alerts import AlertManager
|
from agenteval.intelligent_eval.alerts import AlertManager
|
||||||
|
|
||||||
manager = AlertManager(session)
|
client = OpenClawClient()
|
||||||
|
manager = AlertManager(session, openclaw_client=client)
|
||||||
alerts = manager.check_rules()
|
alerts = manager.check_rules()
|
||||||
|
# T7: link each newly-created alert to auto-scale (best-effort).
|
||||||
|
for alert in alerts:
|
||||||
|
await manager.maybe_autoscale(alert)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"success": True,
|
"success": True,
|
||||||
|
|||||||
@ -38,9 +38,12 @@ def db_session(tmp_db_path: Path) -> Session:
|
|||||||
ExplorationMessageDB,
|
ExplorationMessageDB,
|
||||||
ExplorationSessionDB,
|
ExplorationSessionDB,
|
||||||
IntelligentEvalDB,
|
IntelligentEvalDB,
|
||||||
|
IntelligentEvalDecisionLogDB,
|
||||||
IntelligentEvalMessageDB,
|
IntelligentEvalMessageDB,
|
||||||
IntelligentEvalSessionDB,
|
IntelligentEvalSessionDB,
|
||||||
|
IntelligentEvalTaskQueueDB,
|
||||||
ModelConfigDB,
|
ModelConfigDB,
|
||||||
|
OpenClawCronPoolDB,
|
||||||
ScenarioDB,
|
ScenarioDB,
|
||||||
ScenarioModelBindingDB,
|
ScenarioModelBindingDB,
|
||||||
TurnDB,
|
TurnDB,
|
||||||
|
|||||||
@ -23,6 +23,7 @@ def client(tmp_path):
|
|||||||
IntelligentEvalSessionDB,
|
IntelligentEvalSessionDB,
|
||||||
IntelligentEvalTaskQueueDB,
|
IntelligentEvalTaskQueueDB,
|
||||||
)
|
)
|
||||||
|
from agenteval.intelligent_eval.alerts import AlertHistoryDB # noqa: F401
|
||||||
|
|
||||||
engine = create_engine(
|
engine = create_engine(
|
||||||
f"sqlite:///{tmp_path / 'test.db'}",
|
f"sqlite:///{tmp_path / 'test.db'}",
|
||||||
@ -97,70 +98,44 @@ def test_check_alerts_records_task_backlog_alert(
|
|||||||
assert "task_backlog" in types
|
assert "task_backlog" in types
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.xfail(
|
|
||||||
reason=(
|
|
||||||
"Known gap: no alert→auto-scale link. check_alerts records alerts but "
|
|
||||||
"never invokes auto_scale. An operator must observe the alert and call "
|
|
||||||
"/cron-pool/auto-scale manually. Tracked in "
|
|
||||||
".scratch/v111-architecture-scan.md."
|
|
||||||
),
|
|
||||||
strict=False,
|
|
||||||
)
|
|
||||||
def test_check_alerts_triggers_auto_scale_on_high_utilization(
|
def test_check_alerts_triggers_auto_scale_on_high_utilization(
|
||||||
client: TestClient, db_session: Session
|
client: TestClient, db_session: Session
|
||||||
):
|
):
|
||||||
"""check_alerts must drive auto_scale so high-utilization triggers a scale-up.
|
"""check_alerts must drive auto_scale so an alert fires a scale-up.
|
||||||
|
|
||||||
Today the link is missing: check_alerts is independent of auto_scale.
|
Uses task_backlog (threshold=50, duration_minutes=0 → fires on first
|
||||||
|
call) to avoid the multi-call wait required by pool_utilization
|
||||||
|
(duration_minutes=10).
|
||||||
"""
|
"""
|
||||||
_seed_high_utilization_pool(db_session)
|
|
||||||
|
|
||||||
from agenteval.intelligent_eval import cron_pool as cron_pool_mod
|
from agenteval.intelligent_eval import cron_pool as cron_pool_mod
|
||||||
|
from agenteval.storage.db import (
|
||||||
|
IntelligentEvalDB,
|
||||||
|
IntelligentEvalTaskQueueDB,
|
||||||
|
)
|
||||||
|
|
||||||
|
eval_db = IntelligentEvalDB(
|
||||||
|
name="backlog-autoscale",
|
||||||
|
target_id="t1",
|
||||||
|
status="executing",
|
||||||
|
started_at=utc_now(),
|
||||||
|
)
|
||||||
|
db_session.add(eval_db)
|
||||||
|
db_session.commit()
|
||||||
|
for _ in range(51):
|
||||||
|
db_session.add(
|
||||||
|
IntelligentEvalTaskQueueDB(
|
||||||
|
eval_id=eval_db.id,
|
||||||
|
status="pending",
|
||||||
|
priority=1,
|
||||||
|
reason="slot_due",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
real_scale_up = cron_pool_mod.scale_up
|
real_scale_up = cron_pool_mod.scale_up
|
||||||
cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment]
|
cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment]
|
||||||
try:
|
try:
|
||||||
client.post("/api/openclaw/cron-pool/check-alerts")
|
client.post("/api/openclaw/cron-pool/check-alerts")
|
||||||
client.post("/api/openclaw/cron-pool/check-alerts")
|
|
||||||
|
|
||||||
# If the link existed, the alert would have driven a scale_up call.
|
|
||||||
# Today: no such call. The test asserts the desired behaviour.
|
|
||||||
assert cron_pool_mod.scale_up.await_count >= 1
|
|
||||||
finally:
|
|
||||||
cron_pool_mod.scale_up = real_scale_up
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.xfail(
|
|
||||||
reason=(
|
|
||||||
"Known gap: no alert→auto-scale link. check_alerts records alerts but "
|
|
||||||
"never invokes auto_scale. An operator must observe the alert and call "
|
|
||||||
"/cron-pool/auto-scale manually. Tracked in "
|
|
||||||
".scratch/v111-architecture-scan.md."
|
|
||||||
),
|
|
||||||
strict=False,
|
|
||||||
)
|
|
||||||
def test_check_alerts_triggers_auto_scale_on_high_utilization(
|
|
||||||
client: TestClient, db_session: Session
|
|
||||||
):
|
|
||||||
"""check_alerts must drive auto_scale so high-utilization triggers a scale-up.
|
|
||||||
|
|
||||||
Today the link is missing: check_alerts is independent of auto_scale.
|
|
||||||
"""
|
|
||||||
_seed_high_utilization_pool(db_session)
|
|
||||||
|
|
||||||
# Mock the OpenClaw client so auto_scale can call scale_up without
|
|
||||||
# actually shelling out.
|
|
||||||
from agenteval.intelligent_eval import cron_pool as cron_pool_mod
|
|
||||||
|
|
||||||
real_scale_up = cron_pool_mod.scale_up
|
|
||||||
cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment]
|
|
||||||
try:
|
|
||||||
# Trigger the alert path twice (first sets triggered_at, second fires).
|
|
||||||
client.post("/api/openclaw/cron-pool/check-alerts")
|
|
||||||
client.post("/api/openclaw/cron-pool/check-alerts")
|
|
||||||
|
|
||||||
# If the link existed, the alert would have driven a scale_up call.
|
|
||||||
# Today: no such call. The test asserts the desired behaviour.
|
|
||||||
assert cron_pool_mod.scale_up.await_count >= 1
|
assert cron_pool_mod.scale_up.await_count >= 1
|
||||||
finally:
|
finally:
|
||||||
cron_pool_mod.scale_up = real_scale_up
|
cron_pool_mod.scale_up = real_scale_up
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user