"""Unit tests for fault tolerance and recovery.""" from datetime import timedelta from unittest.mock import AsyncMock, MagicMock import pytest from sqlmodel import Session, select from agenteval.intelligent_eval import fault_tolerance from agenteval.intelligent_eval.openclaw_client import OpenClawClient from agenteval.storage.db import ( IntelligentEvalTaskQueueDB, OpenClawCronPoolDB, utc_now, ) @pytest.fixture() def mock_openclaw_client(): """Mock OpenClaw client.""" client = MagicMock(spec=OpenClawClient) client.list_crons = AsyncMock(return_value=[]) client.sync_cron_states = AsyncMock(return_value=0) return client async def test_detect_and_handle_stuck_crons(db_session: Session, mock_openclaw_client): """Test detecting and handling stuck crons.""" # Create stuck cron cron = OpenClawCronPoolDB( openclaw_cron_id="stuck-cron", status="busy", current_eval_id="eval-1", last_active_at=utc_now() - timedelta(minutes=15), ) db_session.add(cron) task = IntelligentEvalTaskQueueDB( eval_id="eval-1", status="assigned", priority=1, reason="slot_due", assigned_cron_id="stuck-cron", ) db_session.add(task) db_session.commit() # Mock OpenClaw client methods mock_openclaw_client.delete_cron = AsyncMock() mock_openclaw_client.create_cron = AsyncMock(return_value="new-cron") handled = await fault_tolerance.detect_and_handle_stuck_crons(db_session, mock_openclaw_client) assert handled == 1 # Verify task requeued db_session.refresh(task) assert task.status == "failed" assert task.error == "Cron stuck" # Verify new task created new_task = db_session.exec( select(IntelligentEvalTaskQueueDB).where( IntelligentEvalTaskQueueDB.eval_id == "eval-1", IntelligentEvalTaskQueueDB.status == "pending", ) ).first() assert new_task is not None assert new_task.reason == "cron_stuck_retry" async def test_reconcile_state_orphaned_crons(db_session: Session, mock_openclaw_client): """Test reconciliation when platform DB has crons that OpenClaw doesn't.""" # Create cron in DB cron = OpenClawCronPoolDB( openclaw_cron_id="orphan-cron", status="idle", last_active_at=utc_now(), ) db_session.add(cron) db_session.commit() # Mock OpenClaw returning empty list (cron doesn't exist) mock_openclaw_client.list_crons = AsyncMock(return_value=[]) stats = await fault_tolerance.reconcile_state(db_session, mock_openclaw_client) assert stats["orphaned_crons"] == 1 # Verify cron marked as stuck db_session.refresh(cron) assert cron.status == "stuck" async def test_reconcile_state_missing_crons(db_session: Session, mock_openclaw_client): """Test reconciliation when OpenClaw has crons that platform DB doesn't.""" from agenteval.intelligent_eval.openclaw_client import OpenClawCron # Mock OpenClaw returning a cron mock_openclaw_client.list_crons = AsyncMock( return_value=[ OpenClawCron( id="missing-cron", name="worker-1", schedule="* * * * *", enabled=True, state={"status": "idle"}, ) ] ) stats = await fault_tolerance.reconcile_state(db_session, mock_openclaw_client) assert stats["missing_crons"] == 1 # Verify cron synced to DB cron = db_session.exec( select(OpenClawCronPoolDB).where(OpenClawCronPoolDB.openclaw_cron_id == "missing-cron") ).first() assert cron is not None async def test_reconcile_state_requeue_inactive_tasks(db_session: Session, mock_openclaw_client): """Test requeuing tasks assigned to inactive crons.""" # Create task assigned to non-existent cron task = IntelligentEvalTaskQueueDB( eval_id="eval-1", status="assigned", priority=1, reason="slot_due", assigned_cron_id="nonexistent-cron", ) db_session.add(task) db_session.commit() mock_openclaw_client.list_crons = AsyncMock(return_value=[]) stats = await fault_tolerance.reconcile_state(db_session, mock_openclaw_client) assert stats["requeued_tasks"] == 1 # Verify task failed and new task created db_session.refresh(task) assert task.status == "failed" new_task = db_session.exec( select(IntelligentEvalTaskQueueDB).where( IntelligentEvalTaskQueueDB.eval_id == "eval-1", IntelligentEvalTaskQueueDB.status == "pending", ) ).first() assert new_task is not None assert new_task.reason == "cron_inactive_retry" async def test_recover_from_platform_restart(db_session: Session, mock_openclaw_client): """Test recovery from platform restart.""" # Create task assigned to inactive cron task = IntelligentEvalTaskQueueDB( eval_id="eval-1", status="assigned", priority=1, reason="slot_due", assigned_cron_id="inactive-cron", ) db_session.add(task) db_session.commit() mock_openclaw_client.list_crons = AsyncMock(return_value=[]) stats = await fault_tolerance.recover_from_platform_restart(db_session, mock_openclaw_client) assert stats["assigned_tasks_checked"] == 1 assert stats["requeued_tasks"] == 1 # Verify task requeued new_task = db_session.exec( select(IntelligentEvalTaskQueueDB).where( IntelligentEvalTaskQueueDB.eval_id == "eval-1", IntelligentEvalTaskQueueDB.status == "pending", ) ).first() assert new_task is not None assert new_task.reason == "platform_restart_retry" async def test_recover_from_openclaw_restart(db_session: Session, mock_openclaw_client): """Test recovery from OpenClaw restart.""" from agenteval.intelligent_eval.openclaw_client import OpenClawCron # Mock OpenClaw returning crons mock_openclaw_client.list_crons = AsyncMock( return_value=[ OpenClawCron( id="recovered-cron", name="worker-1", schedule="* * * * *", enabled=True, state={"status": "idle"}, ) ] ) stats = await fault_tolerance.recover_from_openclaw_restart(db_session, mock_openclaw_client) assert stats["synced_crons"] == 1 # Verify cron synced to DB cron = db_session.exec( select(OpenClawCronPoolDB).where(OpenClawCronPoolDB.openclaw_cron_id == "recovered-cron") ).first() assert cron is not None