"""Smoke tests for intelligent eval data model (ticket 01).""" import pytest from agenteval.intelligent_eval.models import ( IntelligentEval, IntelligentEvalMessage, IntelligentEvalSession, IntelligentEvalSessionStatus, IntelligentEvalStatus, ) from agenteval.intelligent_eval.repository import ( CompareAndSetStatus, IntelligentEvalMessageRepository, IntelligentEvalRepository, IntelligentEvalSessionRepository, ) from agenteval.storage.db import EvalTargetDB from sqlalchemy.pool import StaticPool from sqlmodel import Session, SQLModel, create_engine @pytest.fixture def db_session(): engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) SQLModel.metadata.create_all(engine) session = Session(engine) target = EvalTargetDB(id="t1", name="test-target") session.add(target) session.commit() yield session session.close() class TestIntelligentEvalRepository: def test_create_and_get(self, db_session): repo = IntelligentEvalRepository(db_session) ev = repo.create( IntelligentEval( name="test-eval", target_id="t1", goal="evaluate customer service", seeds={"personas": [], "goals": []}, intent="test intent", role_description="impatient customer", time_window_hours=24, ) ) assert ev.id is not None assert ev.status == IntelligentEvalStatus.DRAFT assert ev.name == "test-eval" assert ev.time_window_hours == 24 fetched = repo.get(ev.id) assert fetched is not None assert fetched.goal == "evaluate customer service" assert fetched.seeds == {"personas": [], "goals": []} def test_list_all(self, db_session): repo = IntelligentEvalRepository(db_session) repo.create(IntelligentEval(name="eval-1", target_id="t1")) repo.create(IntelligentEval(name="eval-2", target_id="t1")) all_evals = repo.list_all() assert len(all_evals) == 2 def test_status_transitions(self, db_session): repo = IntelligentEvalRepository(db_session) ev = repo.create(IntelligentEval(name="eval", target_id="t1")) assert ev.status == IntelligentEvalStatus.DRAFT ev = repo._compare_and_set_status( ev.id, expected_status=IntelligentEvalStatus.DRAFT, new_status=IntelligentEvalStatus.PLANNING ).evaluation assert ev.status == IntelligentEvalStatus.PLANNING ev = repo._submit_plan_if_planning(ev.id, {"dimensions": ["test"]}).evaluation assert ev.status == IntelligentEvalStatus.PENDING_APPROVAL ev = repo._compare_and_set_status( ev.id, expected_status=IntelligentEvalStatus.PENDING_APPROVAL, new_status=IntelligentEvalStatus.EXECUTING, ).evaluation assert ev.status == IntelligentEvalStatus.EXECUTING assert ev.started_at is not None ev = repo._submit_report_if_executing(ev.id, {"summary": "done"}).evaluation assert ev.status == IntelligentEvalStatus.COMPLETED assert ev.completed_at is not None def test_compare_and_set_status_reports_conflict_without_overwrite(self, db_session): repo = IntelligentEvalRepository(db_session) ev = repo.create(IntelligentEval(name="eval", target_id="t1")) applied = repo._compare_and_set_status( ev.id, expected_status=IntelligentEvalStatus.DRAFT, new_status=IntelligentEvalStatus.PLANNING, ) assert applied.status is CompareAndSetStatus.APPLIED assert applied.evaluation is not None assert applied.evaluation.status is IntelligentEvalStatus.PLANNING conflict = repo._compare_and_set_status( ev.id, expected_status=IntelligentEvalStatus.DRAFT, new_status=IntelligentEvalStatus.EXECUTING, ) assert conflict.status is CompareAndSetStatus.CONFLICT assert repo.get(ev.id).status is IntelligentEvalStatus.PLANNING def test_compare_and_set_status_reports_missing(self, db_session): result = IntelligentEvalRepository(db_session)._compare_and_set_status( "missing", expected_status=IntelligentEvalStatus.DRAFT, new_status=IntelligentEvalStatus.PLANNING, ) assert result.status is CompareAndSetStatus.NOT_FOUND def test_compare_and_set_status_rolls_back_failed_transaction(self, db_session, monkeypatch): repo = IntelligentEvalRepository(db_session) ev = repo.create(IntelligentEval(name="eval", target_id="t1")) def fail_commit(): raise RuntimeError("commit failed") monkeypatch.setattr(db_session, "commit", fail_commit) with pytest.raises(RuntimeError, match="commit failed"): repo._compare_and_set_status( ev.id, expected_status=IntelligentEvalStatus.DRAFT, new_status=IntelligentEvalStatus.PLANNING, ) monkeypatch.undo() assert repo.get(ev.id).status is IntelligentEvalStatus.DRAFT def test_conditional_plan_write_updates_plan_and_status_together(self, db_session): repo = IntelligentEvalRepository(db_session) ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING)) result = repo._submit_plan_if_planning(ev.id, {"dimensions": ["退货"]}) assert result.status is CompareAndSetStatus.APPLIED updated = repo.get(ev.id) assert updated.status is IntelligentEvalStatus.PENDING_APPROVAL assert updated.plan == {"dimensions": ["退货"]} assert updated.plan_feedback is None def test_conditional_plan_write_failure_leaves_state_unchanged(self, db_session, monkeypatch): repo = IntelligentEvalRepository(db_session) ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING)) monkeypatch.setattr(db_session, "commit", lambda: (_ for _ in ()).throw(RuntimeError("commit failed"))) with pytest.raises(RuntimeError, match="commit failed"): repo._submit_plan_if_planning(ev.id, {"dimensions": ["退货"]}) monkeypatch.undo() unchanged = repo.get(ev.id) assert unchanged.status is IntelligentEvalStatus.PLANNING assert unchanged.plan is None def test_plan_and_report_json(self, db_session): repo = IntelligentEvalRepository(db_session) ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING)) plan = {"dimensions": ["退货"], "virtual_users": [], "estimated_sessions": 3} ev = repo._submit_plan_if_planning(ev.id, plan).evaluation assert ev.plan == plan ev = repo._compare_and_set_status( ev.id, expected_status=IntelligentEvalStatus.PENDING_APPROVAL, new_status=IntelligentEvalStatus.EXECUTING, ).evaluation report = {"summary": "good", "findings": []} ev = repo._submit_report_if_executing(ev.id, report).evaluation assert ev.report == report class TestIntelligentEvalSessionRepository: def test_create_and_list(self, db_session): eval_repo = IntelligentEvalRepository(db_session) ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) sess_repo = IntelligentEvalSessionRepository(db_session) sess = sess_repo.create( IntelligentEvalSession( eval_id=ev.id, target_id="t1", persona={"name": "user1", "patience": "low"}, goal="complete return", dimension="退货流程", ) ) assert sess.id is not None assert sess.status == IntelligentEvalSessionStatus.RUNNING assert sess.persona == {"name": "user1", "patience": "low"} sessions = sess_repo.list_by_eval(ev.id) assert len(sessions) == 1 def test_list_by_evals_batches_and_preserves_empty_groups(self, db_session): eval_repo = IntelligentEvalRepository(db_session) first = eval_repo.create(IntelligentEval(name="first", target_id="t1")) second = eval_repo.create(IntelligentEval(name="second", target_id="t1")) sess_repo = IntelligentEvalSessionRepository(db_session) sess_repo.create(IntelligentEvalSession(eval_id=first.id, target_id="t1", goal="first")) grouped = sess_repo.list_by_evals([first.id, second.id, "missing"]) assert [item.goal for item in grouped[first.id]] == ["first"] assert grouped[second.id] == [] assert grouped["missing"] == [] def test_close_session(self, db_session): eval_repo = IntelligentEvalRepository(db_session) ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) sess_repo = IntelligentEvalSessionRepository(db_session) sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1")) verdict = {"goal_achieved": True, "issues": []} closed = sess_repo.close(sess.id, verdict) assert closed.status == IntelligentEvalSessionStatus.COMPLETED assert closed.verdict == verdict assert closed.closed_at is not None def test_increment_turns(self, db_session): eval_repo = IntelligentEvalRepository(db_session) ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) sess_repo = IntelligentEvalSessionRepository(db_session) sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1")) assert sess.turn_count == 0 sess_repo.increment_turns(sess.id) sess_repo.increment_turns(sess.id) updated = sess_repo.get(sess.id) assert updated.turn_count == 2 def test_close_if_running_is_conditional(self, db_session): eval_repo = IntelligentEvalRepository(db_session) ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) sess_repo = IntelligentEvalSessionRepository(db_session) sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1")) status, closed = sess_repo._close_if_running(sess.id, {"goal_achieved": True}) assert status is CompareAndSetStatus.APPLIED assert closed is not None assert closed.status is IntelligentEvalSessionStatus.COMPLETED status, closed = sess_repo._close_if_running(sess.id, {"goal_achieved": False}) assert status is CompareAndSetStatus.CONFLICT assert closed is None def test_create_if_executing_uses_parent_target_and_rejects_other_states(self, db_session): eval_repo = IntelligentEvalRepository(db_session) ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) sess_repo = IntelligentEvalSessionRepository(db_session) status, created = sess_repo._create_if_executing( IntelligentEvalSession(eval_id=ev.id, target_id="wrong", goal="goal") ) assert status is CompareAndSetStatus.CONFLICT assert created is None eval_repo._compare_and_set_status( ev.id, expected_status=IntelligentEvalStatus.DRAFT, new_status=IntelligentEvalStatus.EXECUTING, ) status, created = sess_repo._create_if_executing( IntelligentEvalSession(eval_id=ev.id, target_id="wrong", goal="goal") ) assert status is CompareAndSetStatus.APPLIED assert created is not None assert created.target_id == "t1" def test_user_message_and_turn_count_share_one_transaction(self, db_session): eval_repo = IntelligentEvalRepository(db_session) ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) eval_repo._compare_and_set_status( ev.id, expected_status=IntelligentEvalStatus.DRAFT, new_status=IntelligentEvalStatus.EXECUTING, ) sess_repo = IntelligentEvalSessionRepository(db_session) _, sess = sess_repo._create_if_executing(IntelligentEvalSession(eval_id=ev.id, target_id="t1")) message_repo = IntelligentEvalMessageRepository(db_session) message = IntelligentEvalMessage(session_id=sess.id, content="hello") assert message_repo._create_user_and_increment(message) is CompareAndSetStatus.APPLIED assert sess_repo.get(sess.id).turn_count == 1 assert message_repo.list_by_session(sess.id)[0].content == "hello" class TestIntelligentEvalMessageRepository: def test_create_and_list(self, db_session): eval_repo = IntelligentEvalRepository(db_session) ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) sess_repo = IntelligentEvalSessionRepository(db_session) sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1")) msg_repo = IntelligentEvalMessageRepository(db_session) msg_repo.create(IntelligentEvalMessage(session_id=sess.id, role="user", content="hello")) msg_repo.create(IntelligentEvalMessage(session_id=sess.id, role="assistant", content="hi", latency_ms=120)) messages = msg_repo.list_by_session(sess.id) assert len(messages) == 2 assert messages[0].role == "user" assert messages[1].role == "assistant" assert messages[1].latency_ms == 120