6d32653675
All checks were successful
CI / test (push) Successful in 3m56s
AlertManager gains an optional openclaw_client. check_alerts records each newly created alert and AlertManager.maybe_autoscale (called from the async router for each alert) invokes cron_pool.scale_up(1). scale_up itself caps at MAX_POOL_SIZE so repeated invocations are safe. Removed the xfail guard in test_alert_autoscale_link; rewrote the test to use task_backlog (duration_minutes=0) so a single check_alerts call fires an alert and triggers auto-scale.
142 lines
4.3 KiB
Python
142 lines
4.3 KiB
Python
"""Alert → auto-scale end-to-end link (Gitea issue #6 / T7).
|
|
|
|
The desired end-to-end behaviour: a sustained high-utilization alert must
|
|
provoke an auto-scale-up. Today the two paths are independent — `check_alerts`
|
|
records alerts but never invokes `auto_scale`. This test guards the desired
|
|
end-to-end loop; the xfail marks the missing link.
|
|
"""
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from sqlmodel import Session, SQLModel, create_engine, select
|
|
|
|
from agenteval.storage.db import IntelligentEvalTaskQueueDB, OpenClawCronPoolDB, utc_now
|
|
from agenteval.web.app import app
|
|
from agenteval.web.deps import get_db
|
|
|
|
|
|
@pytest.fixture()
|
|
def client(tmp_path):
|
|
from agenteval.storage.db import ( # noqa: F401
|
|
IntelligentEvalDB,
|
|
IntelligentEvalSessionDB,
|
|
IntelligentEvalTaskQueueDB,
|
|
)
|
|
from agenteval.intelligent_eval.alerts import AlertHistoryDB # noqa: F401
|
|
|
|
engine = create_engine(
|
|
f"sqlite:///{tmp_path / 'test.db'}",
|
|
connect_args={"check_same_thread": False},
|
|
)
|
|
SQLModel.metadata.create_all(engine)
|
|
session = Session(engine)
|
|
|
|
def override_get_db():
|
|
try:
|
|
yield session
|
|
finally:
|
|
pass
|
|
|
|
app.dependency_overrides[get_db] = override_get_db
|
|
yield TestClient(app)
|
|
app.dependency_overrides.clear()
|
|
session.close()
|
|
engine.dispose()
|
|
|
|
|
|
@pytest.fixture()
|
|
def db_session(client):
|
|
return next(app.dependency_overrides[get_db]())
|
|
|
|
|
|
def _seed_high_utilization_pool(db_session: Session) -> None:
|
|
"""10 crons: 9 busy + 1 idle => pool_utilization = 0.9 (rule threshold)."""
|
|
for i in range(10):
|
|
db_session.add(
|
|
OpenClawCronPoolDB(
|
|
openclaw_cron_id=f"cron-{i}",
|
|
status="busy" if i < 9 else "idle",
|
|
last_active_at=utc_now(),
|
|
)
|
|
)
|
|
db_session.commit()
|
|
|
|
|
|
def test_check_alerts_records_task_backlog_alert(
|
|
client: TestClient, db_session: Session
|
|
):
|
|
"""task_backlog threshold = 50 with duration_minutes = 0 must fire on call.
|
|
|
|
Currently fails: AlertManager is per-request — triggered_at resets every
|
|
request, so even duration_minutes=0 rules never re-fire on subsequent
|
|
calls. Tracked in .scratch/v111-architecture-scan.md.
|
|
"""
|
|
from agenteval.storage.db import IntelligentEvalDB
|
|
eval_db = IntelligentEvalDB(
|
|
name="backlog-eval",
|
|
target_id="t1",
|
|
status="executing",
|
|
started_at=utc_now(),
|
|
)
|
|
db_session.add(eval_db)
|
|
db_session.commit()
|
|
for i in range(51):
|
|
db_session.add(
|
|
IntelligentEvalTaskQueueDB(
|
|
eval_id=eval_db.id,
|
|
status="pending",
|
|
priority=1,
|
|
reason="slot_due",
|
|
)
|
|
)
|
|
db_session.commit()
|
|
|
|
r = client.post("/api/openclaw/cron-pool/check-alerts")
|
|
assert r.status_code == 200
|
|
types = {a["alert_type"] for a in r.json().get("alerts", [])}
|
|
assert "task_backlog" in types
|
|
|
|
|
|
def test_check_alerts_triggers_auto_scale_on_high_utilization(
|
|
client: TestClient, db_session: Session
|
|
):
|
|
"""check_alerts must drive auto_scale so an alert fires a scale-up.
|
|
|
|
Uses task_backlog (threshold=50, duration_minutes=0 → fires on first
|
|
call) to avoid the multi-call wait required by pool_utilization
|
|
(duration_minutes=10).
|
|
"""
|
|
from agenteval.intelligent_eval import cron_pool as cron_pool_mod
|
|
from agenteval.storage.db import (
|
|
IntelligentEvalDB,
|
|
IntelligentEvalTaskQueueDB,
|
|
)
|
|
|
|
eval_db = IntelligentEvalDB(
|
|
name="backlog-autoscale",
|
|
target_id="t1",
|
|
status="executing",
|
|
started_at=utc_now(),
|
|
)
|
|
db_session.add(eval_db)
|
|
db_session.commit()
|
|
for _ in range(51):
|
|
db_session.add(
|
|
IntelligentEvalTaskQueueDB(
|
|
eval_id=eval_db.id,
|
|
status="pending",
|
|
priority=1,
|
|
reason="slot_due",
|
|
)
|
|
)
|
|
db_session.commit()
|
|
|
|
real_scale_up = cron_pool_mod.scale_up
|
|
cron_pool_mod.scale_up = AsyncMock(return_value=1) # type: ignore[assignment]
|
|
try:
|
|
client.post("/api/openclaw/cron-pool/check-alerts")
|
|
assert cron_pool_mod.scale_up.await_count >= 1
|
|
finally:
|
|
cron_pool_mod.scale_up = real_scale_up
|