"""Integration test for campaign Run-spawning driven by a manual clock. Drives ``advance_campaign`` at injected clock positions against a compressed-time-scale campaign and asserts the spawned child Runs match the plan (count, ownership, scenario) and complete with results/summary in the DB. No real timer is used — the clock is injected, mirroring how the durable loop (later ticket) will call the same seam. """ import pytest from agenteval.evaluation.campaign_runner import advance_campaign, reconcile_campaign_child_runs from agenteval.models import ( Campaign, CampaignPlanEntry, CampaignSummary, Case, CaseType, ChannelType, EvalTarget, PlatformType, RunStatus, RunTrigger, Scenario, SchedulerState, TargetStatus, ) from agenteval.storage.repository import CampaignRepository, RunRepository, ScenarioRepository, TargetRepository from tests.unit.mock_channel import MockChannel @pytest.fixture() def seeded_db(db_session, monkeypatch): """Point every get_session consumer at the test session, seed target + scenario, and make the channel a MockChannel so no network is hit.""" from agenteval.channels import factory as factory_module from agenteval.evaluation import engine as engine_module from agenteval.storage import db as db_module from agenteval.storage import repository as repo_module def _test_get_session(): return db_session monkeypatch.setattr(db_module, "get_session", _test_get_session) monkeypatch.setattr(repo_module, "get_session", _test_get_session) monkeypatch.setattr(engine_module, "get_session", _test_get_session) channel = MockChannel(reply_delay=0.0) db_session.info["mock_channel"] = channel monkeypatch.setattr(factory_module.ChannelFactory, "create", lambda target: channel) target = EvalTarget( id="t-1", name="mock-target", platform=PlatformType.AI_DIGITAL_EMPLOYEE, channel_type=ChannelType.TUTU_API, channel_config={"base_url": "http://mock", "token": "x"}, status=TargetStatus.ACTIVE, ) TargetRepository(db_session).create(target) scenario = Scenario( id="s-1", name="mock-scenario", cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])], ) ScenarioRepository(db_session).create(scenario) return db_session def _make_campaign(session) -> Campaign: return CampaignRepository(session).create(Campaign( name="compressed", target_id="t-1", status="running", window_seconds=7200, time_scale=3600.0, # 1 real second == 3600 window seconds plan=[ CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2), CampaignPlanEntry(scenario_id="s-1", offset_seconds=3600, count=1), ], )) async def test_advance_spawns_due_runs_matching_plan(seeded_db): campaign = _make_campaign(seeded_db) # t=0 → window offset 0 → only entry 0 (count 2) is due. r0 = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db) assert len(r0.spawned_run_ids) == 2 assert r0.finished is False # t=1s → window offset 3600 → entry 1 (count 1) becomes due. r1 = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=1.0, session=seeded_db) assert len(r1.spawned_run_ids) == 1 runs = RunRepository(seeded_db).list_all() assert len(runs) == 3 assert { (run.campaign_plan_index, run.campaign_occurrence_index) for run in runs } == {(0, 0), (0, 1), (1, 0)} for run in runs: assert run.campaign_id == campaign.id assert run.scenario_id == "s-1" assert run.triggered_by == RunTrigger.CAMPAIGN assert run.status == RunStatus.COMPLETED assert run.summary.total_cases == 1 async def test_advance_is_idempotent(seeded_db): campaign = _make_campaign(seeded_db) await advance_campaign(campaign_id=campaign.id, elapsed_seconds=1.0, session=seeded_db) # Re-advancing to the same clock must not double-spawn. again = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=1.0, session=seeded_db) assert again.spawned_run_ids == [] assert len(RunRepository(seeded_db).list_all()) == 3 # entry0(2) + entry1(1) async def test_advance_reports_finished_at_window_end(seeded_db): campaign = _make_campaign(seeded_db) # t=2s → offset 7200 == window end. result = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=2.0, session=seeded_db) assert result.finished is True async def test_advance_missing_campaign_returns_none(seeded_db): assert await advance_campaign(campaign_id="nope", elapsed_seconds=0.0, session=seeded_db) is None async def test_restart_after_claim_does_not_create_second_run(seeded_db): campaign = CampaignRepository(seeded_db).create( Campaign( name="claimed-before-crash", target_id="t-1", status="running", window_seconds=60, plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)], ) ) claim = RunRepository(seeded_db).claim_campaign_run( campaign_id=campaign.id, scenario_id="s-1", scenario_version=1, plan_index=0, occurrence_index=0, ) assert claim.run is not None result = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db) runs = RunRepository(seeded_db).list_by_campaign(campaign.id) assert result.spawned_run_ids == [] assert len(runs) == 1 assert runs[0].id == claim.run.id assert runs[0].status is RunStatus.PENDING async def test_partial_claim_does_not_hide_remaining_occurrences(seeded_db): campaign = CampaignRepository(seeded_db).create( Campaign( name="partial-claim", target_id="t-1", status="running", window_seconds=60, plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)], ) ) first_claim = RunRepository(seeded_db).claim_campaign_run( campaign_id=campaign.id, scenario_id="s-1", scenario_version=1, plan_index=0, occurrence_index=0, ) assert first_claim.run is not None campaign.summary = CampaignSummary(scheduler=SchedulerState(spawned_indices=[0])) CampaignRepository(seeded_db).update(campaign) await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db) runs = RunRepository(seeded_db).list_by_campaign(campaign.id) assert len(runs) == 2 assert {run.campaign_occurrence_index for run in runs} == {0, 1} assert all(run.status is RunStatus.COMPLETED for run in runs) async def test_partial_spawn_failure_is_retried_until_all_occurrences_are_claimed(seeded_db, monkeypatch): from agenteval.evaluation import campaign_runner campaign = CampaignRepository(seeded_db).create( Campaign( name="partial-failure", target_id="t-1", status="running", window_seconds=60, plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)], ) ) real_spawn = campaign_runner._spawn_child_run failed_once = False async def flaky_spawn(*args, occurrence_index, **kwargs): nonlocal failed_once if occurrence_index == 1 and not failed_once: failed_once = True raise RuntimeError("transient spawn failure") return await real_spawn(*args, occurrence_index=occurrence_index, **kwargs) monkeypatch.setattr(campaign_runner, "_spawn_child_run", flaky_spawn) await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db) after_failure = CampaignRepository(seeded_db).get(campaign.id) assert after_failure.summary.scheduler.spawned_indices == [] await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db) runs = RunRepository(seeded_db).list_by_campaign(campaign.id) recovered = CampaignRepository(seeded_db).get(campaign.id) assert {run.campaign_occurrence_index for run in runs} == {0, 1} assert recovered.summary.scheduler.spawned_indices == [0] async def test_recovery_resumes_pending_claim_without_replacing_identity(seeded_db): campaign = _make_campaign(seeded_db) claim = RunRepository(seeded_db).claim_campaign_run( campaign_id=campaign.id, scenario_id="s-1", scenario_version=1, plan_index=0, occurrence_index=0, ) assert claim.run is not None result = await reconcile_campaign_child_runs(campaign.id, seeded_db) recovered = RunRepository(seeded_db).get(claim.run.id) assert result is not None assert result.resumed_run_ids == [claim.run.id] assert recovered.status is RunStatus.COMPLETED assert len(RunRepository(seeded_db).list_by_campaign(campaign.id)) == 1 async def test_recovery_fails_running_claim_without_replaying_messages(seeded_db): campaign = _make_campaign(seeded_db) repo = RunRepository(seeded_db) claim = repo.claim_campaign_run( campaign_id=campaign.id, scenario_id="s-1", scenario_version=1, plan_index=0, occurrence_index=0, ) assert claim.run is not None claim.run.status = RunStatus.RUNNING repo.update(claim.run) result = await reconcile_campaign_child_runs(campaign.id, seeded_db) interrupted = repo.get(claim.run.id) assert result is not None assert result.resumed_run_ids == [] assert result.failed_run_ids == [claim.run.id] assert interrupted.status is RunStatus.FAILED assert interrupted.summary.error.code == "interrupted" assert seeded_db.info["mock_channel"].send_calls == 0