226 lines
7.9 KiB
Python
226 lines
7.9 KiB
Python
"""Integration test for campaign Run-spawning driven by a manual clock.
|
|
|
|
Drives ``advance_campaign`` at injected clock positions against a
|
|
compressed-time-scale campaign and asserts the spawned child Runs match the
|
|
plan (count, ownership, scenario) and complete with results/summary in the DB.
|
|
No real timer is used — the clock is injected, mirroring how the durable loop
|
|
(later ticket) will call the same seam.
|
|
"""
|
|
|
|
import pytest
|
|
from agenteval.evaluation.campaign_runner import advance_campaign, reconcile_campaign_child_runs
|
|
from agenteval.models import (
|
|
Campaign,
|
|
CampaignPlanEntry,
|
|
Case,
|
|
CaseType,
|
|
ChannelType,
|
|
EvalTarget,
|
|
PlatformType,
|
|
RunStatus,
|
|
RunTrigger,
|
|
Scenario,
|
|
TargetStatus,
|
|
)
|
|
from agenteval.storage.repository import CampaignRepository, RunRepository, ScenarioRepository, TargetRepository
|
|
|
|
from tests.unit.mock_channel import MockChannel
|
|
|
|
|
|
@pytest.fixture()
|
|
def seeded_db(db_session, monkeypatch):
|
|
"""Point every get_session consumer at the test session, seed target +
|
|
scenario, and make the channel a MockChannel so no network is hit."""
|
|
from agenteval.channels import factory as factory_module
|
|
from agenteval.evaluation import engine as engine_module
|
|
from agenteval.storage import db as db_module
|
|
from agenteval.storage import repository as repo_module
|
|
|
|
def _test_get_session():
|
|
return db_session
|
|
|
|
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(engine_module, "get_session", _test_get_session)
|
|
|
|
channel = MockChannel(reply_delay=0.0)
|
|
db_session.info["mock_channel"] = channel
|
|
monkeypatch.setattr(factory_module.ChannelFactory, "create", lambda target: channel)
|
|
|
|
target = EvalTarget(
|
|
id="t-1", name="mock-target",
|
|
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={"base_url": "http://mock", "token": "x"},
|
|
status=TargetStatus.ACTIVE,
|
|
)
|
|
TargetRepository(db_session).create(target)
|
|
|
|
scenario = Scenario(
|
|
id="s-1", name="mock-scenario",
|
|
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
|
)
|
|
ScenarioRepository(db_session).create(scenario)
|
|
|
|
return db_session
|
|
|
|
|
|
def _make_campaign(session) -> Campaign:
|
|
return CampaignRepository(session).create(Campaign(
|
|
name="compressed",
|
|
target_id="t-1",
|
|
status="running",
|
|
window_seconds=7200,
|
|
time_scale=3600.0, # 1 real second == 3600 window seconds
|
|
plan=[
|
|
CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2),
|
|
CampaignPlanEntry(scenario_id="s-1", offset_seconds=3600, count=1),
|
|
],
|
|
))
|
|
|
|
|
|
async def test_advance_spawns_due_runs_matching_plan(seeded_db):
|
|
campaign = _make_campaign(seeded_db)
|
|
|
|
# t=0 → window offset 0 → only entry 0 (count 2) is due.
|
|
r0 = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
|
|
assert len(r0.spawned_run_ids) == 2
|
|
assert r0.finished is False
|
|
|
|
# t=1s → window offset 3600 → entry 1 (count 1) becomes due.
|
|
r1 = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=1.0, session=seeded_db)
|
|
assert len(r1.spawned_run_ids) == 1
|
|
|
|
runs = RunRepository(seeded_db).list_all()
|
|
assert len(runs) == 3
|
|
assert {
|
|
(run.campaign_plan_index, run.campaign_occurrence_index) for run in runs
|
|
} == {(0, 0), (0, 1), (1, 0)}
|
|
for run in runs:
|
|
assert run.campaign_id == campaign.id
|
|
assert run.scenario_id == "s-1"
|
|
assert run.triggered_by == RunTrigger.CAMPAIGN
|
|
assert run.status == RunStatus.COMPLETED
|
|
assert run.summary.total_cases == 1
|
|
|
|
|
|
async def test_advance_is_idempotent(seeded_db):
|
|
campaign = _make_campaign(seeded_db)
|
|
|
|
await advance_campaign(campaign_id=campaign.id, elapsed_seconds=1.0, session=seeded_db)
|
|
# Re-advancing to the same clock must not double-spawn.
|
|
again = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=1.0, session=seeded_db)
|
|
assert again.spawned_run_ids == []
|
|
assert len(RunRepository(seeded_db).list_all()) == 3 # entry0(2) + entry1(1)
|
|
|
|
|
|
async def test_advance_reports_finished_at_window_end(seeded_db):
|
|
campaign = _make_campaign(seeded_db)
|
|
# t=2s → offset 7200 == window end.
|
|
result = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=2.0, session=seeded_db)
|
|
assert result.finished is True
|
|
|
|
|
|
async def test_advance_missing_campaign_returns_none(seeded_db):
|
|
assert await advance_campaign(campaign_id="nope", elapsed_seconds=0.0, session=seeded_db) is None
|
|
|
|
|
|
async def test_restart_after_claim_does_not_create_second_run(seeded_db):
|
|
campaign = CampaignRepository(seeded_db).create(
|
|
Campaign(
|
|
name="claimed-before-crash",
|
|
target_id="t-1",
|
|
status="running",
|
|
window_seconds=60,
|
|
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
|
)
|
|
)
|
|
claim = RunRepository(seeded_db).claim_campaign_run(
|
|
campaign_id=campaign.id,
|
|
scenario_id="s-1",
|
|
scenario_version=1,
|
|
plan_index=0,
|
|
occurrence_index=0,
|
|
)
|
|
assert claim.run is not None
|
|
|
|
result = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
|
|
|
|
runs = RunRepository(seeded_db).list_by_campaign(campaign.id)
|
|
assert result.spawned_run_ids == []
|
|
assert len(runs) == 1
|
|
assert runs[0].id == claim.run.id
|
|
assert runs[0].status is RunStatus.PENDING
|
|
|
|
|
|
async def test_partial_claim_does_not_hide_remaining_occurrences(seeded_db):
|
|
campaign = CampaignRepository(seeded_db).create(
|
|
Campaign(
|
|
name="partial-claim",
|
|
target_id="t-1",
|
|
status="running",
|
|
window_seconds=60,
|
|
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)],
|
|
)
|
|
)
|
|
first_claim = RunRepository(seeded_db).claim_campaign_run(
|
|
campaign_id=campaign.id,
|
|
scenario_id="s-1",
|
|
scenario_version=1,
|
|
plan_index=0,
|
|
occurrence_index=0,
|
|
)
|
|
assert first_claim.run is not None
|
|
|
|
await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
|
|
|
|
runs = RunRepository(seeded_db).list_by_campaign(campaign.id)
|
|
assert len(runs) == 2
|
|
assert {run.campaign_occurrence_index for run in runs} == {0, 1}
|
|
assert all(run.status is RunStatus.COMPLETED for run in runs)
|
|
|
|
|
|
async def test_recovery_resumes_pending_claim_without_replacing_identity(seeded_db):
|
|
campaign = _make_campaign(seeded_db)
|
|
claim = RunRepository(seeded_db).claim_campaign_run(
|
|
campaign_id=campaign.id,
|
|
scenario_id="s-1",
|
|
scenario_version=1,
|
|
plan_index=0,
|
|
occurrence_index=0,
|
|
)
|
|
assert claim.run is not None
|
|
|
|
result = await reconcile_campaign_child_runs(campaign.id, seeded_db)
|
|
|
|
recovered = RunRepository(seeded_db).get(claim.run.id)
|
|
assert result is not None
|
|
assert result.resumed_run_ids == [claim.run.id]
|
|
assert recovered.status is RunStatus.COMPLETED
|
|
assert len(RunRepository(seeded_db).list_by_campaign(campaign.id)) == 1
|
|
|
|
|
|
async def test_recovery_fails_running_claim_without_replaying_messages(seeded_db):
|
|
campaign = _make_campaign(seeded_db)
|
|
repo = RunRepository(seeded_db)
|
|
claim = repo.claim_campaign_run(
|
|
campaign_id=campaign.id,
|
|
scenario_id="s-1",
|
|
scenario_version=1,
|
|
plan_index=0,
|
|
occurrence_index=0,
|
|
)
|
|
assert claim.run is not None
|
|
claim.run.status = RunStatus.RUNNING
|
|
repo.update(claim.run)
|
|
|
|
result = await reconcile_campaign_child_runs(campaign.id, seeded_db)
|
|
|
|
interrupted = repo.get(claim.run.id)
|
|
assert result is not None
|
|
assert result.resumed_run_ids == []
|
|
assert result.failed_run_ids == [claim.run.id]
|
|
assert interrupted.status is RunStatus.FAILED
|
|
assert interrupted.summary.error.code == "interrupted"
|
|
assert seeded_db.info["mock_channel"].send_calls == 0
|