"""Integration tests for fault tolerance and recovery.""" from datetime import timedelta import pytest from fastapi.testclient import TestClient from sqlmodel import Session, SQLModel, create_engine, select from unittest.mock import AsyncMock from agenteval.intelligent_eval.openclaw_client import OpenClawClient, OpenClawCron from agenteval.storage.db import ( IntelligentEvalTaskQueueDB, OpenClawCronPoolDB, utc_now, ) from agenteval.web.app import app from agenteval.web.deps import get_db @pytest.fixture() def client(tmp_path): """Create a TestClient with a fresh database.""" from agenteval.storage.db import ( # noqa: F401 IntelligentEvalTaskQueueDB, OpenClawCronPoolDB, ) engine = create_engine( f"sqlite:///{tmp_path / 'test.db'}", connect_args={"check_same_thread": False}, ) SQLModel.metadata.create_all(engine) session = Session(engine) def override_get_db(): try: yield session finally: pass app.dependency_overrides[get_db] = override_get_db client = TestClient(app) yield client app.dependency_overrides.clear() session.close() engine.dispose() @pytest.fixture() def db_session(client): """Get the database session from the client fixture.""" return next(app.dependency_overrides[get_db]()) @pytest.fixture() def mock_openclaw_client(): """Mock OpenClaw client.""" client = OpenClawClient() client.list_crons = AsyncMock(return_value=[]) client.sync_cron_states = AsyncMock(return_value=0) client.delete_cron = AsyncMock() client.create_cron = AsyncMock(return_value="new-cron") return client def test_stuck_cron_detection_api(client: TestClient, db_session: Session): """Test stuck cron detection via API.""" # Create stuck cron cron = OpenClawCronPoolDB( openclaw_cron_id="stuck-cron", status="busy", current_eval_id="eval-1", last_active_at=utc_now() - timedelta(minutes=15), ) db_session.add(cron) db_session.commit() # Verify cron is stuck from agenteval.intelligent_eval.cron_pool import detect_stuck_crons stuck = detect_stuck_crons(db_session) assert len(stuck) == 1 assert stuck[0].openclaw_cron_id == "stuck-cron" def test_task_requeue_after_cron_stuck(client: TestClient, db_session: Session): """Test task requeue after cron gets stuck.""" # Create stuck cron with assigned task cron = OpenClawCronPoolDB( openclaw_cron_id="stuck-cron", status="busy", current_eval_id="eval-1", last_active_at=utc_now() - timedelta(minutes=15), ) db_session.add(cron) task = IntelligentEvalTaskQueueDB( eval_id="eval-1", status="assigned", priority=1, reason="slot_due", assigned_cron_id="stuck-cron", ) db_session.add(task) db_session.commit() # Simulate handling stuck cron from agenteval.intelligent_eval.cron_pool import handle_stuck_cron from unittest.mock import MagicMock mock_client = MagicMock(spec=OpenClawClient) mock_client.delete_cron = AsyncMock() mock_client.create_cron = AsyncMock(return_value="new-cron") import asyncio asyncio.run(handle_stuck_cron(cron, db_session, mock_client)) # Verify task requeued db_session.refresh(task) assert task.status == "failed" new_task = db_session.exec( select(IntelligentEvalTaskQueueDB).where( IntelligentEvalTaskQueueDB.eval_id == "eval-1", IntelligentEvalTaskQueueDB.status == "pending", ) ).first() assert new_task is not None assert new_task.reason == "cron_stuck_retry" def test_state_reconciliation_api(client: TestClient, db_session: Session): """Test state reconciliation via API.""" # Create orphan cron (exists in DB but not in OpenClaw) cron = OpenClawCronPoolDB( openclaw_cron_id="orphan-cron", status="idle", last_active_at=utc_now(), ) db_session.add(cron) db_session.commit() # Mock OpenClaw client from agenteval.intelligent_eval import fault_tolerance mock_client = OpenClawClient() mock_client.list_crons = AsyncMock(return_value=[]) mock_client.sync_cron_states = AsyncMock(return_value=0) import asyncio stats = asyncio.run(fault_tolerance.reconcile_state(db_session, mock_client)) assert stats["orphaned_crons"] == 1 # Verify cron marked as stuck db_session.refresh(cron) assert cron.status == "stuck" def test_platform_restart_recovery(client: TestClient, db_session: Session): """Test platform restart recovery.""" # Create task assigned to inactive cron task = IntelligentEvalTaskQueueDB( eval_id="eval-1", status="assigned", priority=1, reason="slot_due", assigned_cron_id="inactive-cron", ) db_session.add(task) db_session.commit() # Mock OpenClaw client from agenteval.intelligent_eval import fault_tolerance mock_client = OpenClawClient() mock_client.list_crons = AsyncMock(return_value=[]) import asyncio stats = asyncio.run(fault_tolerance.recover_from_platform_restart(db_session, mock_client)) assert stats["assigned_tasks_checked"] == 1 assert stats["requeued_tasks"] == 1 # Verify task requeued new_task = db_session.exec( select(IntelligentEvalTaskQueueDB).where( IntelligentEvalTaskQueueDB.eval_id == "eval-1", IntelligentEvalTaskQueueDB.status == "pending", ) ).first() assert new_task is not None assert new_task.reason == "platform_restart_retry" def test_openclaw_restart_recovery(client: TestClient, db_session: Session): """Test OpenClaw restart recovery.""" # Mock OpenClaw client returning recovered crons from agenteval.intelligent_eval import fault_tolerance mock_client = OpenClawClient() mock_client.list_crons = AsyncMock( return_value=[ OpenClawCron( id="recovered-cron", name="worker-1", schedule="* * * * *", enabled=True, state={"status": "idle"}, ) ] ) import asyncio stats = asyncio.run(fault_tolerance.recover_from_openclaw_restart(db_session, mock_client)) assert stats["synced_crons"] == 1 # Verify cron synced to DB cron = db_session.exec( select(OpenClawCronPoolDB).where(OpenClawCronPoolDB.openclaw_cron_id == "recovered-cron") ).first() assert cron is not None