AgentEvalTool/tests/integration/test_fault_tolerance_e2e.py

"""Integration tests for fault tolerance and recovery."""

from datetime import timedelta

import pytest
from fastapi.testclient import TestClient
from sqlmodel import Session, SQLModel, create_engine, select
from unittest.mock import AsyncMock

from agenteval.intelligent_eval.openclaw_client import OpenClawClient, OpenClawCron
from agenteval.storage.db import (
    IntelligentEvalTaskQueueDB,
    OpenClawCronPoolDB,
    utc_now,
)
from agenteval.web.app import app
from agenteval.web.deps import get_db


@pytest.fixture()
def client(tmp_path):
    """Create a TestClient with a fresh database."""
    from agenteval.storage.db import (  # noqa: F401
        IntelligentEvalTaskQueueDB,
        OpenClawCronPoolDB,
    )

    engine = create_engine(
        f"sqlite:///{tmp_path / 'test.db'}",
        connect_args={"check_same_thread": False},
    )
    SQLModel.metadata.create_all(engine)
    session = Session(engine)

    def override_get_db():
        try:
            yield session
        finally:
            pass

    app.dependency_overrides[get_db] = override_get_db
    client = TestClient(app)
    yield client
    app.dependency_overrides.clear()
    session.close()
    engine.dispose()


@pytest.fixture()
def db_session(client):
    """Get the database session from the client fixture."""
    return next(app.dependency_overrides[get_db]())


@pytest.fixture()
def mock_openclaw_client():
    """Mock OpenClaw client."""
    client = OpenClawClient()
    client.list_crons = AsyncMock(return_value=[])
    client.sync_cron_states = AsyncMock(return_value=0)
    client.delete_cron = AsyncMock()
    client.create_cron = AsyncMock(return_value="new-cron")
    return client


def test_stuck_cron_detection_api(client: TestClient, db_session: Session):
    """Test stuck cron detection via API."""
    # Create stuck cron
    cron = OpenClawCronPoolDB(
        openclaw_cron_id="stuck-cron",
        status="busy",
        current_eval_id="eval-1",
        last_active_at=utc_now() - timedelta(minutes=15),
    )
    db_session.add(cron)
    db_session.commit()

    # Verify cron is stuck
    from agenteval.intelligent_eval.cron_pool import detect_stuck_crons

    stuck = detect_stuck_crons(db_session)
    assert len(stuck) == 1
    assert stuck[0].openclaw_cron_id == "stuck-cron"


def test_task_requeue_after_cron_stuck(client: TestClient, db_session: Session):
    """Test task requeue after cron gets stuck."""
    # Create stuck cron with assigned task
    cron = OpenClawCronPoolDB(
        openclaw_cron_id="stuck-cron",
        status="busy",
        current_eval_id="eval-1",
        last_active_at=utc_now() - timedelta(minutes=15),
    )
    db_session.add(cron)

    task = IntelligentEvalTaskQueueDB(
        eval_id="eval-1",
        status="assigned",
        priority=1,
        reason="slot_due",
        assigned_cron_id="stuck-cron",
    )
    db_session.add(task)
    db_session.commit()

    # Simulate handling stuck cron
    from agenteval.intelligent_eval.cron_pool import handle_stuck_cron
    from unittest.mock import MagicMock

    mock_client = MagicMock(spec=OpenClawClient)
    mock_client.delete_cron = AsyncMock()
    mock_client.create_cron = AsyncMock(return_value="new-cron")

    import asyncio

    asyncio.run(handle_stuck_cron(cron, db_session, mock_client))

    # Verify task requeued
    db_session.refresh(task)
    assert task.status == "failed"

    new_task = db_session.exec(
        select(IntelligentEvalTaskQueueDB).where(
            IntelligentEvalTaskQueueDB.eval_id == "eval-1",
            IntelligentEvalTaskQueueDB.status == "pending",
        )
    ).first()
    assert new_task is not None
    assert new_task.reason == "cron_stuck_retry"


def test_state_reconciliation_api(client: TestClient, db_session: Session):
    """Test state reconciliation via API."""
    # Create orphan cron (exists in DB but not in OpenClaw)
    cron = OpenClawCronPoolDB(
        openclaw_cron_id="orphan-cron",
        status="idle",
        last_active_at=utc_now(),
    )
    db_session.add(cron)
    db_session.commit()

    # Mock OpenClaw client
    from agenteval.intelligent_eval import fault_tolerance

    mock_client = OpenClawClient()
    mock_client.list_crons = AsyncMock(return_value=[])
    mock_client.sync_cron_states = AsyncMock(return_value=0)

    import asyncio

    stats = asyncio.run(fault_tolerance.reconcile_state(db_session, mock_client))

    assert stats["orphaned_crons"] == 1

    # Verify cron marked as stuck
    db_session.refresh(cron)
    assert cron.status == "stuck"


def test_platform_restart_recovery(client: TestClient, db_session: Session):
    """Test platform restart recovery."""
    # Create task assigned to inactive cron
    task = IntelligentEvalTaskQueueDB(
        eval_id="eval-1",
        status="assigned",
        priority=1,
        reason="slot_due",
        assigned_cron_id="inactive-cron",
    )
    db_session.add(task)
    db_session.commit()

    # Mock OpenClaw client
    from agenteval.intelligent_eval import fault_tolerance

    mock_client = OpenClawClient()
    mock_client.list_crons = AsyncMock(return_value=[])

    import asyncio

    stats = asyncio.run(fault_tolerance.recover_from_platform_restart(db_session, mock_client))

    assert stats["assigned_tasks_checked"] == 1
    assert stats["requeued_tasks"] == 1

    # Verify task requeued
    new_task = db_session.exec(
        select(IntelligentEvalTaskQueueDB).where(
            IntelligentEvalTaskQueueDB.eval_id == "eval-1",
            IntelligentEvalTaskQueueDB.status == "pending",
        )
    ).first()
    assert new_task is not None
    assert new_task.reason == "platform_restart_retry"


def test_openclaw_restart_recovery(client: TestClient, db_session: Session):
    """Test OpenClaw restart recovery."""
    # Mock OpenClaw client returning recovered crons
    from agenteval.intelligent_eval import fault_tolerance

    mock_client = OpenClawClient()
    mock_client.list_crons = AsyncMock(
        return_value=[
            OpenClawCron(
                id="recovered-cron",
                name="worker-1",
                schedule="* * * * *",
                enabled=True,
                state={"status": "idle"},
            )
        ]
    )

    import asyncio

    stats = asyncio.run(fault_tolerance.recover_from_openclaw_restart(db_session, mock_client))

    assert stats["synced_crons"] == 1

    # Verify cron synced to DB
    cron = db_session.exec(
        select(OpenClawCronPoolDB).where(OpenClawCronPoolDB.openclaw_cron_id == "recovered-cron")
    ).first()
    assert cron is not None