316 lines
13 KiB
Python
316 lines
13 KiB
Python
"""Smoke tests for intelligent eval data model (ticket 01)."""
|
|
|
|
import pytest
|
|
from agenteval.intelligent_eval.models import (
|
|
IntelligentEval,
|
|
IntelligentEvalMessage,
|
|
IntelligentEvalSession,
|
|
IntelligentEvalSessionStatus,
|
|
IntelligentEvalStatus,
|
|
)
|
|
from agenteval.intelligent_eval.repository import (
|
|
CompareAndSetStatus,
|
|
IntelligentEvalMessageRepository,
|
|
IntelligentEvalRepository,
|
|
IntelligentEvalSessionRepository,
|
|
)
|
|
from agenteval.storage.db import EvalTargetDB
|
|
from sqlalchemy.pool import StaticPool
|
|
from sqlmodel import Session, SQLModel, create_engine
|
|
|
|
|
|
@pytest.fixture
|
|
def db_session():
|
|
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
|
SQLModel.metadata.create_all(engine)
|
|
session = Session(engine)
|
|
target = EvalTargetDB(id="t1", name="test-target")
|
|
session.add(target)
|
|
session.commit()
|
|
yield session
|
|
session.close()
|
|
|
|
|
|
class TestIntelligentEvalRepository:
|
|
def test_create_and_get(self, db_session):
|
|
repo = IntelligentEvalRepository(db_session)
|
|
ev = repo.create(
|
|
IntelligentEval(
|
|
name="test-eval",
|
|
target_id="t1",
|
|
goal="evaluate customer service",
|
|
seeds={"personas": [], "goals": []},
|
|
intent="test intent",
|
|
role_description="impatient customer",
|
|
time_window_hours=24,
|
|
)
|
|
)
|
|
assert ev.id is not None
|
|
assert ev.status == IntelligentEvalStatus.DRAFT
|
|
assert ev.name == "test-eval"
|
|
assert ev.time_window_hours == 24
|
|
|
|
fetched = repo.get(ev.id)
|
|
assert fetched is not None
|
|
assert fetched.goal == "evaluate customer service"
|
|
assert fetched.seeds == {"personas": [], "goals": []}
|
|
|
|
def test_list_all(self, db_session):
|
|
repo = IntelligentEvalRepository(db_session)
|
|
repo.create(IntelligentEval(name="eval-1", target_id="t1"))
|
|
repo.create(IntelligentEval(name="eval-2", target_id="t1"))
|
|
all_evals = repo.list_all()
|
|
assert len(all_evals) == 2
|
|
|
|
def test_status_transitions(self, db_session):
|
|
repo = IntelligentEvalRepository(db_session)
|
|
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
|
|
assert ev.status == IntelligentEvalStatus.DRAFT
|
|
|
|
ev = repo._compare_and_set_status(
|
|
ev.id, expected_status=IntelligentEvalStatus.DRAFT, new_status=IntelligentEvalStatus.PLANNING
|
|
).evaluation
|
|
assert ev.status == IntelligentEvalStatus.PLANNING
|
|
|
|
ev = repo._submit_plan_if_planning(ev.id, {"dimensions": ["test"]}).evaluation
|
|
assert ev.status == IntelligentEvalStatus.PENDING_APPROVAL
|
|
|
|
ev = repo._compare_and_set_status(
|
|
ev.id,
|
|
expected_status=IntelligentEvalStatus.PENDING_APPROVAL,
|
|
new_status=IntelligentEvalStatus.EXECUTING,
|
|
).evaluation
|
|
assert ev.status == IntelligentEvalStatus.EXECUTING
|
|
assert ev.started_at is not None
|
|
|
|
ev = repo._submit_report_if_executing(ev.id, {"summary": "done"}).evaluation
|
|
assert ev.status == IntelligentEvalStatus.COMPLETED
|
|
assert ev.completed_at is not None
|
|
|
|
def test_compare_and_set_status_reports_conflict_without_overwrite(self, db_session):
|
|
repo = IntelligentEvalRepository(db_session)
|
|
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
|
|
|
|
applied = repo._compare_and_set_status(
|
|
ev.id,
|
|
expected_status=IntelligentEvalStatus.DRAFT,
|
|
new_status=IntelligentEvalStatus.PLANNING,
|
|
)
|
|
assert applied.status is CompareAndSetStatus.APPLIED
|
|
assert applied.evaluation is not None
|
|
assert applied.evaluation.status is IntelligentEvalStatus.PLANNING
|
|
|
|
conflict = repo._compare_and_set_status(
|
|
ev.id,
|
|
expected_status=IntelligentEvalStatus.DRAFT,
|
|
new_status=IntelligentEvalStatus.EXECUTING,
|
|
)
|
|
assert conflict.status is CompareAndSetStatus.CONFLICT
|
|
assert repo.get(ev.id).status is IntelligentEvalStatus.PLANNING
|
|
|
|
def test_compare_and_set_status_reports_missing(self, db_session):
|
|
result = IntelligentEvalRepository(db_session)._compare_and_set_status(
|
|
"missing",
|
|
expected_status=IntelligentEvalStatus.DRAFT,
|
|
new_status=IntelligentEvalStatus.PLANNING,
|
|
)
|
|
assert result.status is CompareAndSetStatus.NOT_FOUND
|
|
|
|
def test_compare_and_set_status_rolls_back_failed_transaction(self, db_session, monkeypatch):
|
|
repo = IntelligentEvalRepository(db_session)
|
|
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
|
|
|
|
def fail_commit():
|
|
raise RuntimeError("commit failed")
|
|
|
|
monkeypatch.setattr(db_session, "commit", fail_commit)
|
|
with pytest.raises(RuntimeError, match="commit failed"):
|
|
repo._compare_and_set_status(
|
|
ev.id,
|
|
expected_status=IntelligentEvalStatus.DRAFT,
|
|
new_status=IntelligentEvalStatus.PLANNING,
|
|
)
|
|
|
|
monkeypatch.undo()
|
|
assert repo.get(ev.id).status is IntelligentEvalStatus.DRAFT
|
|
|
|
def test_conditional_plan_write_updates_plan_and_status_together(self, db_session):
|
|
repo = IntelligentEvalRepository(db_session)
|
|
ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING))
|
|
|
|
result = repo._submit_plan_if_planning(ev.id, {"dimensions": ["退货"]})
|
|
|
|
assert result.status is CompareAndSetStatus.APPLIED
|
|
updated = repo.get(ev.id)
|
|
assert updated.status is IntelligentEvalStatus.PENDING_APPROVAL
|
|
assert updated.plan == {"dimensions": ["退货"]}
|
|
assert updated.plan_feedback is None
|
|
|
|
def test_conditional_plan_write_failure_leaves_state_unchanged(self, db_session, monkeypatch):
|
|
repo = IntelligentEvalRepository(db_session)
|
|
ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING))
|
|
|
|
monkeypatch.setattr(db_session, "commit", lambda: (_ for _ in ()).throw(RuntimeError("commit failed")))
|
|
with pytest.raises(RuntimeError, match="commit failed"):
|
|
repo._submit_plan_if_planning(ev.id, {"dimensions": ["退货"]})
|
|
|
|
monkeypatch.undo()
|
|
unchanged = repo.get(ev.id)
|
|
assert unchanged.status is IntelligentEvalStatus.PLANNING
|
|
assert unchanged.plan is None
|
|
|
|
def test_plan_and_report_json(self, db_session):
|
|
repo = IntelligentEvalRepository(db_session)
|
|
ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING))
|
|
|
|
plan = {"dimensions": ["退货"], "virtual_users": [], "estimated_sessions": 3}
|
|
ev = repo._submit_plan_if_planning(ev.id, plan).evaluation
|
|
assert ev.plan == plan
|
|
|
|
ev = repo._compare_and_set_status(
|
|
ev.id,
|
|
expected_status=IntelligentEvalStatus.PENDING_APPROVAL,
|
|
new_status=IntelligentEvalStatus.EXECUTING,
|
|
).evaluation
|
|
report = {"summary": "good", "findings": []}
|
|
ev = repo._submit_report_if_executing(ev.id, report).evaluation
|
|
assert ev.report == report
|
|
|
|
|
|
class TestIntelligentEvalSessionRepository:
|
|
def test_create_and_list(self, db_session):
|
|
eval_repo = IntelligentEvalRepository(db_session)
|
|
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
|
|
|
sess_repo = IntelligentEvalSessionRepository(db_session)
|
|
sess = sess_repo.create(
|
|
IntelligentEvalSession(
|
|
eval_id=ev.id,
|
|
target_id="t1",
|
|
persona={"name": "user1", "patience": "low"},
|
|
goal="complete return",
|
|
dimension="退货流程",
|
|
)
|
|
)
|
|
assert sess.id is not None
|
|
assert sess.status == IntelligentEvalSessionStatus.RUNNING
|
|
assert sess.persona == {"name": "user1", "patience": "low"}
|
|
|
|
sessions = sess_repo.list_by_eval(ev.id)
|
|
assert len(sessions) == 1
|
|
|
|
def test_list_by_evals_batches_and_preserves_empty_groups(self, db_session):
|
|
eval_repo = IntelligentEvalRepository(db_session)
|
|
first = eval_repo.create(IntelligentEval(name="first", target_id="t1"))
|
|
second = eval_repo.create(IntelligentEval(name="second", target_id="t1"))
|
|
|
|
sess_repo = IntelligentEvalSessionRepository(db_session)
|
|
sess_repo.create(IntelligentEvalSession(eval_id=first.id, target_id="t1", goal="first"))
|
|
|
|
grouped = sess_repo.list_by_evals([first.id, second.id, "missing"])
|
|
|
|
assert [item.goal for item in grouped[first.id]] == ["first"]
|
|
assert grouped[second.id] == []
|
|
assert grouped["missing"] == []
|
|
|
|
def test_close_session(self, db_session):
|
|
eval_repo = IntelligentEvalRepository(db_session)
|
|
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
|
|
|
sess_repo = IntelligentEvalSessionRepository(db_session)
|
|
sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
|
|
|
|
verdict = {"goal_achieved": True, "issues": []}
|
|
closed = sess_repo.close(sess.id, verdict)
|
|
assert closed.status == IntelligentEvalSessionStatus.COMPLETED
|
|
assert closed.verdict == verdict
|
|
assert closed.closed_at is not None
|
|
|
|
def test_increment_turns(self, db_session):
|
|
eval_repo = IntelligentEvalRepository(db_session)
|
|
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
|
|
|
sess_repo = IntelligentEvalSessionRepository(db_session)
|
|
sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
|
|
assert sess.turn_count == 0
|
|
|
|
sess_repo.increment_turns(sess.id)
|
|
sess_repo.increment_turns(sess.id)
|
|
updated = sess_repo.get(sess.id)
|
|
assert updated.turn_count == 2
|
|
|
|
def test_close_if_running_is_conditional(self, db_session):
|
|
eval_repo = IntelligentEvalRepository(db_session)
|
|
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
|
|
|
sess_repo = IntelligentEvalSessionRepository(db_session)
|
|
sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
|
|
status, closed = sess_repo._close_if_running(sess.id, {"goal_achieved": True})
|
|
|
|
assert status is CompareAndSetStatus.APPLIED
|
|
assert closed is not None
|
|
assert closed.status is IntelligentEvalSessionStatus.COMPLETED
|
|
|
|
status, closed = sess_repo._close_if_running(sess.id, {"goal_achieved": False})
|
|
assert status is CompareAndSetStatus.CONFLICT
|
|
assert closed is None
|
|
|
|
def test_create_if_executing_uses_parent_target_and_rejects_other_states(self, db_session):
|
|
eval_repo = IntelligentEvalRepository(db_session)
|
|
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
|
sess_repo = IntelligentEvalSessionRepository(db_session)
|
|
|
|
status, created = sess_repo._create_if_executing(
|
|
IntelligentEvalSession(eval_id=ev.id, target_id="wrong", goal="goal")
|
|
)
|
|
assert status is CompareAndSetStatus.CONFLICT
|
|
assert created is None
|
|
|
|
eval_repo._compare_and_set_status(
|
|
ev.id,
|
|
expected_status=IntelligentEvalStatus.DRAFT,
|
|
new_status=IntelligentEvalStatus.EXECUTING,
|
|
)
|
|
status, created = sess_repo._create_if_executing(
|
|
IntelligentEvalSession(eval_id=ev.id, target_id="wrong", goal="goal")
|
|
)
|
|
assert status is CompareAndSetStatus.APPLIED
|
|
assert created is not None
|
|
assert created.target_id == "t1"
|
|
|
|
def test_user_message_and_turn_count_share_one_transaction(self, db_session):
|
|
eval_repo = IntelligentEvalRepository(db_session)
|
|
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
|
eval_repo._compare_and_set_status(
|
|
ev.id,
|
|
expected_status=IntelligentEvalStatus.DRAFT,
|
|
new_status=IntelligentEvalStatus.EXECUTING,
|
|
)
|
|
sess_repo = IntelligentEvalSessionRepository(db_session)
|
|
_, sess = sess_repo._create_if_executing(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
|
|
|
|
message_repo = IntelligentEvalMessageRepository(db_session)
|
|
message = IntelligentEvalMessage(session_id=sess.id, content="hello")
|
|
assert message_repo._create_user_and_increment(message) is CompareAndSetStatus.APPLIED
|
|
assert sess_repo.get(sess.id).turn_count == 1
|
|
assert message_repo.list_by_session(sess.id)[0].content == "hello"
|
|
|
|
|
|
class TestIntelligentEvalMessageRepository:
|
|
def test_create_and_list(self, db_session):
|
|
eval_repo = IntelligentEvalRepository(db_session)
|
|
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
|
|
|
sess_repo = IntelligentEvalSessionRepository(db_session)
|
|
sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
|
|
|
|
msg_repo = IntelligentEvalMessageRepository(db_session)
|
|
msg_repo.create(IntelligentEvalMessage(session_id=sess.id, role="user", content="hello"))
|
|
msg_repo.create(IntelligentEvalMessage(session_id=sess.id, role="assistant", content="hi", latency_ms=120))
|
|
|
|
messages = msg_repo.list_by_session(sess.id)
|
|
assert len(messages) == 2
|
|
assert messages[0].role == "user"
|
|
assert messages[1].role == "assistant"
|
|
assert messages[1].latency_ms == 120
|