"""End-to-end test for intelligent eval with cron pool.""" from datetime import timedelta import pytest from sqlmodel import Session, select from agenteval.intelligent_eval import task_queue from agenteval.intelligent_eval.decision import DecisionType, is_eval_completed, make_decision from agenteval.intelligent_eval.models import IntelligentEvalStatus from agenteval.storage.db import ( IntelligentEvalDB, IntelligentEvalDecisionLogDB, IntelligentEvalSessionDB, IntelligentEvalTaskQueueDB, utc_now, ) def test_end_to_end_eval_lifecycle(db_session: Session): """Test end-to-end eval lifecycle: create -> scan -> decide -> execute -> complete.""" # 1. Create eval eval_db = IntelligentEvalDB( name="test-eval", target_id="target1", status=IntelligentEvalStatus.EXECUTING.value, started_at=utc_now() - timedelta(hours=8, minutes=30), ) eval_db.set_plan({ "time_distribution": [{"time_slot": "8-10h", "sessions": 2}], "estimated_sessions": 2, }) db_session.add(eval_db) db_session.commit() # 2. Scan and enqueue tasks enqueued = task_queue.scan_and_enqueue_tasks(db_session) assert enqueued == 1 # Verify task created task = db_session.exec( select(IntelligentEvalTaskQueueDB).where(IntelligentEvalTaskQueueDB.eval_id == eval_db.id) ).first() assert task is not None assert task.status == "pending" assert task.reason == "slot_due" # 3. Worker makes decision (should be EXECUTE_SESSION) decision = make_decision(eval_db, db_session) assert decision.decision_type == DecisionType.EXECUTE_SESSION assert "欠账" in decision.reason # Record decision log log = IntelligentEvalDecisionLogDB( eval_id=eval_db.id, decision_type=decision.decision_type.value, reason=decision.reason, cron_id="cron-123", ) log.set_context(decision.context) db_session.add(log) db_session.commit() # 4. Execute session (simulate) session_db = IntelligentEvalSessionDB( eval_id=eval_db.id, target_id="target1", status="running", created_at=utc_now() - timedelta(minutes=20), ) db_session.add(session_db) db_session.commit() # 5. Complete session session_db.status = "completed" session_db.set_verdict({"severity": "medium", "issues": ["minor issue"]}) db_session.commit() # 6. Worker makes decision again (should still be EXECUTE_SESSION, deficit = 1) decision2 = make_decision(eval_db, db_session) assert decision2.decision_type == DecisionType.EXECUTE_SESSION assert "欠账 1 个会话" in decision2.reason # 7. Execute second session session_db2 = IntelligentEvalSessionDB( eval_id=eval_db.id, target_id="target1", status="running", created_at=utc_now() - timedelta(minutes=10), ) db_session.add(session_db2) db_session.commit() session_db2.status = "completed" session_db2.set_verdict({"severity": "low", "issues": []}) db_session.commit() # 8. Worker makes decision again (should be START_ANALYSIS) decision3 = make_decision(eval_db, db_session) assert decision3.decision_type == DecisionType.START_ANALYSIS assert "所有 2 个会话已完成" in decision3.reason # 9. Submit report eval_db.set_report({"summary": "Test report", "findings": []}) db_session.commit() # 10. Check if eval is completed assert is_eval_completed(eval_db, db_session) is True # 11. Verify decision logs logs = db_session.exec( select(IntelligentEvalDecisionLogDB) .where(IntelligentEvalDecisionLogDB.eval_id == eval_db.id) .order_by(IntelligentEvalDecisionLogDB.created_at) ).all() assert len(logs) >= 1 # At least one decision log def test_decision_logs_complete_history(db_session: Session): """Test that decision logs capture complete history.""" eval_db = IntelligentEvalDB( name="test-eval", target_id="target1", status=IntelligentEvalStatus.EXECUTING.value, started_at=utc_now() - timedelta(hours=8, minutes=30), ) eval_db.set_plan({ "time_distribution": [{"time_slot": "8-10h", "sessions": 2}], "estimated_sessions": 2, }) db_session.add(eval_db) db_session.commit() # Simulate multiple worker ticks decisions_made = [] # Tick 1: No sessions yet decision1 = make_decision(eval_db, db_session) decisions_made.append(decision1) log1 = IntelligentEvalDecisionLogDB( eval_id=eval_db.id, decision_type=decision1.decision_type.value, reason=decision1.reason, cron_id="cron-123", ) log1.set_context(decision1.context) db_session.add(log1) db_session.commit() # Tick 2: One session created session_db = IntelligentEvalSessionDB( eval_id=eval_db.id, target_id="target1", status="running", created_at=utc_now() - timedelta(minutes=20), ) db_session.add(session_db) db_session.commit() decision2 = make_decision(eval_db, db_session) decisions_made.append(decision2) log2 = IntelligentEvalDecisionLogDB( eval_id=eval_db.id, decision_type=decision2.decision_type.value, reason=decision2.reason, cron_id="cron-123", ) log2.set_context(decision2.context) db_session.add(log2) db_session.commit() # Verify logs logs = db_session.exec( select(IntelligentEvalDecisionLogDB) .where(IntelligentEvalDecisionLogDB.eval_id == eval_db.id) .order_by(IntelligentEvalDecisionLogDB.created_at) ).all() assert len(logs) == 2 assert logs[0].decision_type == DecisionType.EXECUTE_SESSION.value assert logs[1].decision_type == DecisionType.EXECUTE_SESSION.value assert logs[0].get_context()["deficit"] == 2 assert logs[1].get_context()["deficit"] == 1