c6b102a9b5
Add the pure scheduling seam (campaign_scheduler.decide_schedule) that, given a static plan and window-clock offset, decides which plan entries are due and whether the window ended — mirroring judgement.combine_case_outcome, with time_scale confined to the clock mapping so it never touches judgement/report. The campaign_runner shell maps injected elapsed time to a window offset, spawns due child Runs through the existing EvalEngine.run(existing_run=...) path with campaign_id + RunTrigger.CAMPAIGN, and persists spawned-entry indices per entry for idempotent, restart-recoverable progress. No auto loop yet (ticket 03).
112 lines
4.3 KiB
Python
112 lines
4.3 KiB
Python
"""Integration test for campaign Run-spawning driven by a manual clock.
|
|
|
|
Drives ``advance_campaign`` at injected clock positions against a
|
|
compressed-time-scale campaign and asserts the spawned child Runs match the
|
|
plan (count, ownership, scenario) and complete with results/summary in the DB.
|
|
No real timer is used — the clock is injected, mirroring how the durable loop
|
|
(later ticket) will call the same seam.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from agenteval.evaluation.campaign_runner import advance_campaign
|
|
from agenteval.models import (
|
|
Campaign, CampaignPlanEntry, Case, CaseType, ChannelType, EvalTarget,
|
|
PlatformType, RunStatus, RunTrigger, Scenario, TargetStatus,
|
|
)
|
|
from agenteval.storage.repository import CampaignRepository, RunRepository, ScenarioRepository, TargetRepository
|
|
from tests.unit.mock_channel import MockChannel
|
|
|
|
|
|
@pytest.fixture()
|
|
def seeded_db(db_session, monkeypatch):
|
|
"""Point every get_session consumer at the test session, seed target +
|
|
scenario, and make the channel a MockChannel so no network is hit."""
|
|
from agenteval.channels import factory as factory_module
|
|
from agenteval.evaluation import engine as engine_module
|
|
from agenteval.storage import db as db_module
|
|
from agenteval.storage import repository as repo_module
|
|
|
|
def _test_get_session():
|
|
return db_session
|
|
|
|
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
|
monkeypatch.setattr(engine_module, "get_session", _test_get_session)
|
|
|
|
channel = MockChannel(reply_delay=0.0)
|
|
monkeypatch.setattr(factory_module.ChannelFactory, "create", lambda target: channel)
|
|
|
|
target = EvalTarget(
|
|
id="t-1", name="mock-target",
|
|
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={"base_url": "http://mock", "token": "x"},
|
|
status=TargetStatus.ACTIVE,
|
|
)
|
|
TargetRepository(db_session).create(target)
|
|
|
|
scenario = Scenario(
|
|
id="s-1", name="mock-scenario",
|
|
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
|
)
|
|
ScenarioRepository(db_session).create(scenario)
|
|
|
|
return db_session
|
|
|
|
|
|
def _make_campaign(session) -> Campaign:
|
|
return CampaignRepository(session).create(Campaign(
|
|
name="compressed",
|
|
target_id="t-1",
|
|
window_seconds=7200,
|
|
time_scale=3600.0, # 1 real second == 3600 window seconds
|
|
plan=[
|
|
CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2),
|
|
CampaignPlanEntry(scenario_id="s-1", offset_seconds=3600, count=1),
|
|
],
|
|
))
|
|
|
|
|
|
async def test_advance_spawns_due_runs_matching_plan(seeded_db):
|
|
campaign = _make_campaign(seeded_db)
|
|
|
|
# t=0 → window offset 0 → only entry 0 (count 2) is due.
|
|
r0 = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
|
|
assert len(r0.spawned_run_ids) == 2
|
|
assert r0.finished is False
|
|
|
|
# t=1s → window offset 3600 → entry 1 (count 1) becomes due.
|
|
r1 = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=1.0, session=seeded_db)
|
|
assert len(r1.spawned_run_ids) == 1
|
|
|
|
runs = RunRepository(seeded_db).list_all()
|
|
assert len(runs) == 3
|
|
for run in runs:
|
|
assert run.campaign_id == campaign.id
|
|
assert run.scenario_id == "s-1"
|
|
assert run.triggered_by == RunTrigger.CAMPAIGN
|
|
assert run.status == RunStatus.COMPLETED
|
|
assert run.summary["total_cases"] == 1
|
|
|
|
|
|
async def test_advance_is_idempotent(seeded_db):
|
|
campaign = _make_campaign(seeded_db)
|
|
|
|
await advance_campaign(campaign_id=campaign.id, elapsed_seconds=1.0, session=seeded_db)
|
|
# Re-advancing to the same clock must not double-spawn.
|
|
again = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=1.0, session=seeded_db)
|
|
assert again.spawned_run_ids == []
|
|
assert len(RunRepository(seeded_db).list_all()) == 3 # entry0(2) + entry1(1)
|
|
|
|
|
|
async def test_advance_reports_finished_at_window_end(seeded_db):
|
|
campaign = _make_campaign(seeded_db)
|
|
# t=2s → offset 7200 == window end.
|
|
result = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=2.0, session=seeded_db)
|
|
assert result.finished is True
|
|
|
|
|
|
async def test_advance_missing_campaign_returns_none(seeded_db):
|
|
assert await advance_campaign(campaign_id="nope", elapsed_seconds=0.0, session=seeded_db) is None
|