AgentEvalTool/backend/agenteval/intelligent_eval/fault_tolerance.py

"""Fault tolerance and recovery for cron pool (故障恢复).

Handles:
- Stuck cron detection and cleanup
- State reconciliation (platform DB vs OpenClaw state)
- Platform restart recovery
- OpenClaw restart recovery
"""

import logging
from datetime import timedelta

from sqlmodel import Session, select

from agenteval.intelligent_eval import cron_pool
from agenteval.intelligent_eval.openclaw_client import OpenClawClient
from agenteval.storage.db import (
    IntelligentEvalTaskQueueDB,
    OpenClawCronPoolDB,
    utc_now,
)

_logger = logging.getLogger("agenteval")


async def detect_and_handle_stuck_crons(session: Session, client: OpenClawClient) -> int:
    """Detect and handle stuck crons.

    Returns:
        Number of stuck crons handled
    """
    stuck_crons = cron_pool.detect_stuck_crons(session)

    for cron in stuck_crons:
        await cron_pool.handle_stuck_cron(cron, session, client)

    if stuck_crons:
        _logger.info(f"Handled {len(stuck_crons)} stuck crons")

    return len(stuck_crons)


async def reconcile_state(session: Session, client: OpenClawClient) -> dict:
    """Reconcile platform DB state with OpenClaw state.

    Checks:
    1. Platform DB has crons that OpenClaw doesn't → mark as stuck
    2. OpenClaw has crons that platform DB doesn't → sync to DB
    3. Assigned tasks have inactive crons → requeue tasks

    Returns:
        Dict with reconciliation stats
    """
    stats = {
        "orphaned_crons": 0,
        "missing_crons": 0,
        "requeued_tasks": 0,
    }

    # Get all crons from both sides
    db_crons = session.exec(select(OpenClawCronPoolDB)).all()
    openclaw_crons = await client.list_crons()
    openclaw_cron_ids = {c.id for c in openclaw_crons}

    # Check 1: Platform DB has crons that OpenClaw doesn't
    for db_cron in db_crons:
        if db_cron.openclaw_cron_id not in openclaw_cron_ids:
            _logger.warning(f"Cron {db_cron.openclaw_cron_id} exists in DB but not in OpenClaw")
            db_cron.status = "stuck"
            db_cron.updated_at = utc_now()
            stats["orphaned_crons"] += 1

    session.commit()

    # Check 2: OpenClaw has crons that platform DB doesn't
    synced = await cron_pool.sync_cron_states(session, client)
    stats["missing_crons"] = synced

    # Check 3: Assigned tasks have inactive crons
    assigned_tasks = session.exec(
        select(IntelligentEvalTaskQueueDB).where(IntelligentEvalTaskQueueDB.status == "assigned")
    ).all()

    for task in assigned_tasks:
        if task.assigned_cron_id is None:
            continue

        # Check if cron is still active
        cron = session.exec(
            select(OpenClawCronPoolDB).where(
                OpenClawCronPoolDB.openclaw_cron_id == task.assigned_cron_id
            )
        ).first()

        if cron is None or cron.status == "stuck":
            _logger.warning(f"Task {task.id} assigned to inactive cron {task.assigned_cron_id}")

            # Mark task as failed
            from agenteval.intelligent_eval.task_queue import complete_task

            complete_task(task.id, False, "Cron inactive", session)

            # Requeue task
            new_task = IntelligentEvalTaskQueueDB(
                eval_id=task.eval_id,
                status="pending",
                priority=1,  # High priority
                reason="cron_inactive_retry",
            )
            session.add(new_task)
            stats["requeued_tasks"] += 1

    session.commit()

    if stats["orphaned_crons"] or stats["missing_crons"] or stats["requeued_tasks"]:
        _logger.info(f"State reconciliation: {stats}")

    return stats


async def recover_from_platform_restart(session: Session, client: OpenClawClient) -> dict:
    """Recover from platform restart.

    Scans all assigned tasks and checks if their crons are still active.
    If not, requeues the tasks.

    Returns:
        Dict with recovery stats
    """
    stats = {
        "assigned_tasks_checked": 0,
        "requeued_tasks": 0,
    }

    # Get all assigned tasks
    assigned_tasks = session.exec(
        select(IntelligentEvalTaskQueueDB).where(IntelligentEvalTaskQueueDB.status == "assigned")
    ).all()

    stats["assigned_tasks_checked"] = len(assigned_tasks)

    for task in assigned_tasks:
        if task.assigned_cron_id is None:
            continue

        # Check if cron exists and is active
        cron = session.exec(
            select(OpenClawCronPoolDB).where(
                OpenClawCronPoolDB.openclaw_cron_id == task.assigned_cron_id
            )
        ).first()

        # Check if cron is active (heartbeat within last 5 minutes)
        if cron:
            threshold = utc_now() - timedelta(minutes=5)
            if cron.last_active_at < threshold:
                cron = None  # Treat as inactive

        if cron is None:
            _logger.info(f"Requeuing task {task.id} (cron inactive after restart)")

            # Mark task as failed
            from agenteval.intelligent_eval.task_queue import complete_task

            complete_task(task.id, False, "Platform restart", session)

            # Requeue task
            new_task = IntelligentEvalTaskQueueDB(
                eval_id=task.eval_id,
                status="pending",
                priority=1,
                reason="platform_restart_retry",
            )
            session.add(new_task)
            stats["requeued_tasks"] += 1

    session.commit()

    if stats["requeued_tasks"]:
        _logger.info(f"Platform restart recovery: {stats}")

    return stats


async def recover_from_openclaw_restart(session: Session, client: OpenClawClient) -> dict:
    """Recover from OpenClaw restart.

    OpenClaw crons persist their state in SQLite, so they should resume
    automatically. This function syncs the state to platform DB.

    Returns:
        Dict with recovery stats
    """
    # Sync cron states from OpenClaw to platform DB
    synced = await cron_pool.sync_cron_states(session, client)

    stats = {
        "synced_crons": synced,
    }

    if synced:
        _logger.info(f"OpenClaw restart recovery: {stats}")

    return stats