AgentEvalTool/tests/integration/test_intelligent_eval_e2e.py

"""End-to-end test for intelligent eval with cron pool."""

from datetime import timedelta

import pytest
from sqlmodel import Session, select

from agenteval.intelligent_eval import task_queue
from agenteval.intelligent_eval.decision import DecisionType, is_eval_completed, make_decision
from agenteval.intelligent_eval.models import IntelligentEvalStatus
from agenteval.storage.db import (
    IntelligentEvalDB,
    IntelligentEvalDecisionLogDB,
    IntelligentEvalSessionDB,
    IntelligentEvalTaskQueueDB,
    utc_now,
)


def test_end_to_end_eval_lifecycle(db_session: Session):
    """Test end-to-end eval lifecycle: create -> scan -> decide -> execute -> complete."""
    # 1. Create eval
    eval_db = IntelligentEvalDB(
        name="test-eval",
        target_id="target1",
        status=IntelligentEvalStatus.EXECUTING.value,
        started_at=utc_now() - timedelta(hours=8, minutes=30),
    )
    eval_db.set_plan({
        "time_distribution": [{"time_slot": "8-10h", "sessions": 2}],
        "estimated_sessions": 2,
    })
    db_session.add(eval_db)
    db_session.commit()

    # 2. Scan and enqueue tasks
    enqueued = task_queue.scan_and_enqueue_tasks(db_session)
    assert enqueued == 1

    # Verify task created
    task = db_session.exec(
        select(IntelligentEvalTaskQueueDB).where(IntelligentEvalTaskQueueDB.eval_id == eval_db.id)
    ).first()
    assert task is not None
    assert task.status == "pending"
    assert task.reason == "slot_due"

    # 3. Worker makes decision (should be EXECUTE_SESSION)
    decision = make_decision(eval_db, db_session)
    assert decision.decision_type == DecisionType.EXECUTE_SESSION
    assert "欠账" in decision.reason

    # Record decision log
    log = IntelligentEvalDecisionLogDB(
        eval_id=eval_db.id,
        decision_type=decision.decision_type.value,
        reason=decision.reason,
        cron_id="cron-123",
    )
    log.set_context(decision.context)
    db_session.add(log)
    db_session.commit()

    # 4. Execute session (simulate)
    session_db = IntelligentEvalSessionDB(
        eval_id=eval_db.id,
        target_id="target1",
        status="running",
        created_at=utc_now() - timedelta(minutes=20),
    )
    db_session.add(session_db)
    db_session.commit()

    # 5. Complete session
    session_db.status = "completed"
    session_db.set_verdict({"severity": "medium", "issues": ["minor issue"]})
    db_session.commit()

    # 6. Worker makes decision again (should still be EXECUTE_SESSION, deficit = 1)
    decision2 = make_decision(eval_db, db_session)
    assert decision2.decision_type == DecisionType.EXECUTE_SESSION
    assert "欠账 1 个会话" in decision2.reason

    # 7. Execute second session
    session_db2 = IntelligentEvalSessionDB(
        eval_id=eval_db.id,
        target_id="target1",
        status="running",
        created_at=utc_now() - timedelta(minutes=10),
    )
    db_session.add(session_db2)
    db_session.commit()

    session_db2.status = "completed"
    session_db2.set_verdict({"severity": "low", "issues": []})
    db_session.commit()

    # 8. Worker makes decision again (should be START_ANALYSIS)
    decision3 = make_decision(eval_db, db_session)
    assert decision3.decision_type == DecisionType.START_ANALYSIS
    assert "所有 2 个会话已完成" in decision3.reason

    # 9. Submit report
    eval_db.set_report({"summary": "Test report", "findings": []})
    db_session.commit()

    # 10. Check if eval is completed
    assert is_eval_completed(eval_db, db_session) is True

    # 11. Verify decision logs
    logs = db_session.exec(
        select(IntelligentEvalDecisionLogDB)
        .where(IntelligentEvalDecisionLogDB.eval_id == eval_db.id)
        .order_by(IntelligentEvalDecisionLogDB.created_at)
    ).all()
    assert len(logs) >= 1  # At least one decision log


def test_decision_logs_complete_history(db_session: Session):
    """Test that decision logs capture complete history."""
    eval_db = IntelligentEvalDB(
        name="test-eval",
        target_id="target1",
        status=IntelligentEvalStatus.EXECUTING.value,
        started_at=utc_now() - timedelta(hours=8, minutes=30),
    )
    eval_db.set_plan({
        "time_distribution": [{"time_slot": "8-10h", "sessions": 2}],
        "estimated_sessions": 2,
    })
    db_session.add(eval_db)
    db_session.commit()

    # Simulate multiple worker ticks
    decisions_made = []

    # Tick 1: No sessions yet
    decision1 = make_decision(eval_db, db_session)
    decisions_made.append(decision1)
    log1 = IntelligentEvalDecisionLogDB(
        eval_id=eval_db.id,
        decision_type=decision1.decision_type.value,
        reason=decision1.reason,
        cron_id="cron-123",
    )
    log1.set_context(decision1.context)
    db_session.add(log1)
    db_session.commit()

    # Tick 2: One session created
    session_db = IntelligentEvalSessionDB(
        eval_id=eval_db.id,
        target_id="target1",
        status="running",
        created_at=utc_now() - timedelta(minutes=20),
    )
    db_session.add(session_db)
    db_session.commit()

    decision2 = make_decision(eval_db, db_session)
    decisions_made.append(decision2)
    log2 = IntelligentEvalDecisionLogDB(
        eval_id=eval_db.id,
        decision_type=decision2.decision_type.value,
        reason=decision2.reason,
        cron_id="cron-123",
    )
    log2.set_context(decision2.context)
    db_session.add(log2)
    db_session.commit()

    # Verify logs
    logs = db_session.exec(
        select(IntelligentEvalDecisionLogDB)
        .where(IntelligentEvalDecisionLogDB.eval_id == eval_db.id)
        .order_by(IntelligentEvalDecisionLogDB.created_at)
    ).all()

    assert len(logs) == 2
    assert logs[0].decision_type == DecisionType.EXECUTE_SESSION.value
    assert logs[1].decision_type == DecisionType.EXECUTE_SESSION.value
    assert logs[0].get_context()["deficit"] == 2
    assert logs[1].get_context()["deficit"] == 1