AgentEvalTool/tests/integration/test_alert_autoscale_link.py

"""Alert → auto-scale end-to-end link (Gitea issue #6 / T7).

The desired end-to-end behaviour: a sustained high-utilization alert must
provoke an auto-scale-up. Today the two paths are independent — `check_alerts`
records alerts but never invokes `auto_scale`. This test guards the desired
end-to-end loop; the xfail marks the missing link.
"""
from unittest.mock import AsyncMock, MagicMock

import pytest
from fastapi.testclient import TestClient
from sqlmodel import Session, SQLModel, create_engine, select

from agenteval.storage.db import IntelligentEvalTaskQueueDB, OpenClawCronPoolDB, utc_now
from agenteval.web.app import app
from agenteval.web.deps import get_db


@pytest.fixture()
def client(tmp_path):
    from agenteval.storage.db import (  # noqa: F401
        IntelligentEvalDB,
        IntelligentEvalSessionDB,
        IntelligentEvalTaskQueueDB,
    )

    engine = create_engine(
        f"sqlite:///{tmp_path / 'test.db'}",
        connect_args={"check_same_thread": False},
    )
    SQLModel.metadata.create_all(engine)
    session = Session(engine)

    def override_get_db():
        try:
            yield session
        finally:
            pass

    app.dependency_overrides[get_db] = override_get_db
    yield TestClient(app)
    app.dependency_overrides.clear()
    session.close()
    engine.dispose()


@pytest.fixture()
def db_session(client):
    return next(app.dependency_overrides[get_db]())


def _seed_high_utilization_pool(db_session: Session) -> None:
    """10 crons: 9 busy + 1 idle => pool_utilization = 0.9 (rule threshold)."""
    for i in range(10):
        db_session.add(
            OpenClawCronPoolDB(
                openclaw_cron_id=f"cron-{i}",
                status="busy" if i < 9 else "idle",
                last_active_at=utc_now(),
            )
        )
    db_session.commit()


def test_check_alerts_records_task_backlog_alert(
    client: TestClient, db_session: Session
):
    """task_backlog threshold = 50 with duration_minutes = 0 must fire on call.

    Currently fails: AlertManager is per-request — triggered_at resets every
    request, so even duration_minutes=0 rules never re-fire on subsequent
    calls. Tracked in .scratch/v111-architecture-scan.md.
    """
    from agenteval.storage.db import IntelligentEvalDB
    eval_db = IntelligentEvalDB(
        name="backlog-eval",
        target_id="t1",
        status="executing",
        started_at=utc_now(),
    )
    db_session.add(eval_db)
    db_session.commit()
    for i in range(51):
        db_session.add(
            IntelligentEvalTaskQueueDB(
                eval_id=eval_db.id,
                status="pending",
                priority=1,
                reason="slot_due",
            )
        )
    db_session.commit()

    r = client.post("/api/openclaw/cron-pool/check-alerts")
    assert r.status_code == 200
    types = {a["alert_type"] for a in r.json().get("alerts", [])}
    assert "task_backlog" in types


@pytest.mark.xfail(
    reason=(
        "Known gap: no alert→auto-scale link. check_alerts records alerts but "
        "never invokes auto_scale. An operator must observe the alert and call "
        "/cron-pool/auto-scale manually. Tracked in "
        ".scratch/v111-architecture-scan.md."
    ),
    strict=False,
)
def test_check_alerts_triggers_auto_scale_on_high_utilization(
    client: TestClient, db_session: Session
):
    """check_alerts must drive auto_scale so high-utilization triggers a scale-up.

    Today the link is missing: check_alerts is independent of auto_scale.
    """
    _seed_high_utilization_pool(db_session)

    from agenteval.intelligent_eval import cron_pool as cron_pool_mod

    real_scale_up = cron_pool_mod.scale_up
    cron_pool_mod.scale_up = AsyncMock(return_value=1)  # type: ignore[assignment]
    try:
        client.post("/api/openclaw/cron-pool/check-alerts")
        client.post("/api/openclaw/cron-pool/check-alerts")

        # If the link existed, the alert would have driven a scale_up call.
        # Today: no such call. The test asserts the desired behaviour.
        assert cron_pool_mod.scale_up.await_count >= 1
    finally:
        cron_pool_mod.scale_up = real_scale_up


@pytest.mark.xfail(
    reason=(
        "Known gap: no alert→auto-scale link. check_alerts records alerts but "
        "never invokes auto_scale. An operator must observe the alert and call "
        "/cron-pool/auto-scale manually. Tracked in "
        ".scratch/v111-architecture-scan.md."
    ),
    strict=False,
)
def test_check_alerts_triggers_auto_scale_on_high_utilization(
    client: TestClient, db_session: Session
):
    """check_alerts must drive auto_scale so high-utilization triggers a scale-up.

    Today the link is missing: check_alerts is independent of auto_scale.
    """
    _seed_high_utilization_pool(db_session)

    # Mock the OpenClaw client so auto_scale can call scale_up without
    # actually shelling out.
    from agenteval.intelligent_eval import cron_pool as cron_pool_mod

    real_scale_up = cron_pool_mod.scale_up
    cron_pool_mod.scale_up = AsyncMock(return_value=1)  # type: ignore[assignment]
    try:
        # Trigger the alert path twice (first sets triggered_at, second fires).
        client.post("/api/openclaw/cron-pool/check-alerts")
        client.post("/api/openclaw/cron-pool/check-alerts")

        # If the link existed, the alert would have driven a scale_up call.
        # Today: no such call. The test asserts the desired behaviour.
        assert cron_pool_mod.scale_up.await_count >= 1
    finally:
        cron_pool_mod.scale_up = real_scale_up