AgentEvalTool/tests/unit/test_fault_tolerance.py

"""Unit tests for fault tolerance and recovery."""

from datetime import timedelta
from unittest.mock import AsyncMock, MagicMock

import pytest
from sqlmodel import Session, select

from agenteval.intelligent_eval import fault_tolerance
from agenteval.intelligent_eval.openclaw_client import OpenClawClient
from agenteval.storage.db import (
    IntelligentEvalTaskQueueDB,
    OpenClawCronPoolDB,
    utc_now,
)


@pytest.fixture()
def mock_openclaw_client():
    """Mock OpenClaw client."""
    client = MagicMock(spec=OpenClawClient)
    client.list_crons = AsyncMock(return_value=[])
    client.sync_cron_states = AsyncMock(return_value=0)
    return client


async def test_detect_and_handle_stuck_crons(db_session: Session, mock_openclaw_client):
    """Test detecting and handling stuck crons."""
    # Create stuck cron
    cron = OpenClawCronPoolDB(
        openclaw_cron_id="stuck-cron",
        status="busy",
        current_eval_id="eval-1",
        last_active_at=utc_now() - timedelta(minutes=15),
    )
    db_session.add(cron)

    task = IntelligentEvalTaskQueueDB(
        eval_id="eval-1",
        status="assigned",
        priority=1,
        reason="slot_due",
        assigned_cron_id="stuck-cron",
    )
    db_session.add(task)
    db_session.commit()

    # Mock OpenClaw client methods
    mock_openclaw_client.delete_cron = AsyncMock()
    mock_openclaw_client.create_cron = AsyncMock(return_value="new-cron")

    handled = await fault_tolerance.detect_and_handle_stuck_crons(db_session, mock_openclaw_client)
    assert handled == 1

    # Verify task requeued
    db_session.refresh(task)
    assert task.status == "failed"
    assert task.error == "Cron stuck"

    # Verify new task created
    new_task = db_session.exec(
        select(IntelligentEvalTaskQueueDB).where(
            IntelligentEvalTaskQueueDB.eval_id == "eval-1",
            IntelligentEvalTaskQueueDB.status == "pending",
        )
    ).first()
    assert new_task is not None
    assert new_task.reason == "cron_stuck_retry"


async def test_reconcile_state_orphaned_crons(db_session: Session, mock_openclaw_client):
    """Test reconciliation when platform DB has crons that OpenClaw doesn't."""
    # Create cron in DB
    cron = OpenClawCronPoolDB(
        openclaw_cron_id="orphan-cron",
        status="idle",
        last_active_at=utc_now(),
    )
    db_session.add(cron)
    db_session.commit()

    # Mock OpenClaw returning empty list (cron doesn't exist)
    mock_openclaw_client.list_crons = AsyncMock(return_value=[])

    stats = await fault_tolerance.reconcile_state(db_session, mock_openclaw_client)
    assert stats["orphaned_crons"] == 1

    # Verify cron marked as stuck
    db_session.refresh(cron)
    assert cron.status == "stuck"


async def test_reconcile_state_missing_crons(db_session: Session, mock_openclaw_client):
    """Test reconciliation when OpenClaw has crons that platform DB doesn't."""
    from agenteval.intelligent_eval.openclaw_client import OpenClawCron

    # Mock OpenClaw returning a cron
    mock_openclaw_client.list_crons = AsyncMock(
        return_value=[
            OpenClawCron(
                id="missing-cron",
                name="worker-1",
                schedule="* * * * *",
                enabled=True,
                state={"status": "idle"},
            )
        ]
    )

    stats = await fault_tolerance.reconcile_state(db_session, mock_openclaw_client)
    assert stats["missing_crons"] == 1

    # Verify cron synced to DB
    cron = db_session.exec(
        select(OpenClawCronPoolDB).where(OpenClawCronPoolDB.openclaw_cron_id == "missing-cron")
    ).first()
    assert cron is not None


async def test_reconcile_state_requeue_inactive_tasks(db_session: Session, mock_openclaw_client):
    """Test requeuing tasks assigned to inactive crons."""
    # Create task assigned to non-existent cron
    task = IntelligentEvalTaskQueueDB(
        eval_id="eval-1",
        status="assigned",
        priority=1,
        reason="slot_due",
        assigned_cron_id="nonexistent-cron",
    )
    db_session.add(task)
    db_session.commit()

    mock_openclaw_client.list_crons = AsyncMock(return_value=[])

    stats = await fault_tolerance.reconcile_state(db_session, mock_openclaw_client)
    assert stats["requeued_tasks"] == 1

    # Verify task failed and new task created
    db_session.refresh(task)
    assert task.status == "failed"

    new_task = db_session.exec(
        select(IntelligentEvalTaskQueueDB).where(
            IntelligentEvalTaskQueueDB.eval_id == "eval-1",
            IntelligentEvalTaskQueueDB.status == "pending",
        )
    ).first()
    assert new_task is not None
    assert new_task.reason == "cron_inactive_retry"


async def test_recover_from_platform_restart(db_session: Session, mock_openclaw_client):
    """Test recovery from platform restart."""
    # Create task assigned to inactive cron
    task = IntelligentEvalTaskQueueDB(
        eval_id="eval-1",
        status="assigned",
        priority=1,
        reason="slot_due",
        assigned_cron_id="inactive-cron",
    )
    db_session.add(task)
    db_session.commit()

    mock_openclaw_client.list_crons = AsyncMock(return_value=[])

    stats = await fault_tolerance.recover_from_platform_restart(db_session, mock_openclaw_client)
    assert stats["assigned_tasks_checked"] == 1
    assert stats["requeued_tasks"] == 1

    # Verify task requeued
    new_task = db_session.exec(
        select(IntelligentEvalTaskQueueDB).where(
            IntelligentEvalTaskQueueDB.eval_id == "eval-1",
            IntelligentEvalTaskQueueDB.status == "pending",
        )
    ).first()
    assert new_task is not None
    assert new_task.reason == "platform_restart_retry"


async def test_recover_from_openclaw_restart(db_session: Session, mock_openclaw_client):
    """Test recovery from OpenClaw restart."""
    from agenteval.intelligent_eval.openclaw_client import OpenClawCron

    # Mock OpenClaw returning crons
    mock_openclaw_client.list_crons = AsyncMock(
        return_value=[
            OpenClawCron(
                id="recovered-cron",
                name="worker-1",
                schedule="* * * * *",
                enabled=True,
                state={"status": "idle"},
            )
        ]
    )

    stats = await fault_tolerance.recover_from_openclaw_restart(db_session, mock_openclaw_client)
    assert stats["synced_crons"] == 1

    # Verify cron synced to DB
    cron = db_session.exec(
        select(OpenClawCronPoolDB).where(OpenClawCronPoolDB.openclaw_cron_id == "recovered-cron")
    ).first()
    assert cron is not None