"""Campaign runner — the side-effect shell around the pure scheduler. Given a campaign and a clock position (real elapsed seconds), it asks ``campaign_scheduler.decide_schedule`` what is due, then spawns those child Runs by reusing the existing single-run execution path (``EvalEngine.run`` with an ``existing_run`` that carries ``campaign_id``). Spawned plan-entry indices are persisted on the campaign so re-advancing the same clock never double-spawns. This module also hosts the durable scheduler loop: a thin async shell that, tick by tick, maps real wall-clock elapsed time (since the campaign's persisted ``started_at``) to a window offset and calls ``advance_campaign``. All authority lives in the DB (window start, spawned progress, status), so the loop can be torn down and rebuilt on restart without losing or duplicating work. """ import asyncio import logging from dataclasses import dataclass, field from typing import Optional from sqlmodel import Session from agenteval.evaluation.campaign_scheduler import ( TickAction, clock_offset, decide_schedule, decide_tick, elapsed_seconds, resolve_finalize, ) from agenteval.evaluation.engine import EvalEngine from agenteval.models import Campaign, CampaignStatus, EvalRun, RunStatus, RunTrigger from agenteval.storage.db import get_session, utc_now from agenteval.storage.repository import ( CampaignRepository, RunRepository, ScenarioRepository, TargetRepository, ) from agenteval.task_registry import TaskRegistry _SPAWNED_KEY = "spawned_indices" _ERRORS_KEY = "errors" # Real wall-clock seconds between scheduler ticks. time_scale compresses the # *window*, not the tick cadence — production 24h campaigns still tick slowly. DEFAULT_TICK_SECONDS = 1.0 _logger = logging.getLogger("agenteval") # Live loop tasks + cooperative cancel events, keyed by campaign id. Authority # is the DB; this registry only holds the in-process handles for the loop. campaign_registry = TaskRegistry() @dataclass class AdvanceResult: """Outcome of advancing a campaign's clock once.""" spawned_run_ids: list[str] = field(default_factory=list) finished: bool = False def _spawned_indices(campaign: Campaign) -> set[int]: scheduler = (campaign.summary or {}).get("scheduler", {}) return set(scheduler.get(_SPAWNED_KEY, [])) def current_window_offset(campaign: Campaign) -> float: """The campaign's live window position (seconds), clamped to the window. RUNNING campaigns derive it from the persisted ``started_at`` and ``time_scale``; not-yet-started campaigns report 0. """ if campaign.started_at is None: return 0.0 offset = clock_offset( elapsed_seconds=elapsed_seconds(now=utc_now(), started_at=campaign.started_at), time_scale=campaign.time_scale, ) return min(offset, float(campaign.window_seconds)) def campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict: """Live progress of a campaign, derived from its child Runs. ``completed_runs`` counts only COMPLETED child Runs; failures stay out of this field (pass_rate semantics are ADR-0002's concern, not this counter). """ return { "current_offset_seconds": current_window_offset(campaign), "spawned_runs": len(runs), "completed_runs": sum(1 for r in runs if r.status == RunStatus.COMPLETED), } async def _spawn_child_run(campaign: Campaign, scenario_id: str, session: Session) -> str: """Create a Run owned by the campaign and drive it through the engine. The Run row is created on the caller's ``session``, but the engine runs on its own session (obtained internally, closed when the run ends) so one child Run's lifecycle never closes the campaign's session — the same isolation the single-run background task uses. """ target = TargetRepository(session).get(campaign.target_id) scenario = ScenarioRepository(session).get(scenario_id) if not target or not scenario: raise ValueError(f"campaign target or scenario missing: {campaign.target_id}/{scenario_id}") run = RunRepository(session).create( EvalRun( target_id=campaign.target_id, scenario_id=scenario_id, scenario_version=scenario.version or 1, campaign_id=campaign.id, triggered_by=RunTrigger.CAMPAIGN, status=RunStatus.PENDING, ) ) engine = EvalEngine(target=target, scenario=scenario, triggered_by=RunTrigger.CAMPAIGN) await engine.run(existing_run=run) return run.id or "" async def advance_campaign( *, campaign_id: str, elapsed_seconds: float, session: Session, cancel_event: Optional[asyncio.Event] = None, ) -> Optional[AdvanceResult]: """Advance the campaign clock to ``elapsed_seconds`` and spawn due Runs. ``elapsed_seconds`` is real wall-clock time since the window started; it is mapped to a window offset via ``time_scale``. A due entry whose spawning raises is skipped and recorded (so one bad entry never wedges the loop), but still marked spawned to avoid infinite retries. Returns ``None`` if the campaign does not exist. """ repo = CampaignRepository(session) campaign = repo.get(campaign_id) if not campaign: return None offset = clock_offset(elapsed_seconds=elapsed_seconds, time_scale=campaign.time_scale) spawned = _spawned_indices(campaign) decision = decide_schedule( plan=campaign.plan, window_seconds=campaign.window_seconds, clock_offset_seconds=offset, spawned_indices=spawned, ) result = AdvanceResult(finished=decision.finished) errors: list[dict] = list((campaign.summary or {}).get("scheduler", {}).get(_ERRORS_KEY, [])) for due in decision.due: # Cancellation stops further spawning; runs already in flight finish. if cancel_event is not None and cancel_event.is_set(): break try: for _ in range(due.entry.count): run_id = await _spawn_child_run(campaign, due.entry.scenario_id, session) result.spawned_run_ids.append(run_id) except Exception as exc: # one entry failing must not wedge the campaign errors.append({"entry_index": due.index, "error": str(exc)}) _logger.warning("活动 %s 计划条目 %d 派生失败（跳过）: %s", campaign_id, due.index, exc) spawned.add(due.index) # Persist progress per entry: a failure partway through a multi-entry # advance must never lose which entries already spawned, since restart # recovery reads this back from the DB. scheduler_state: dict = {_SPAWNED_KEY: sorted(spawned)} if errors: scheduler_state[_ERRORS_KEY] = errors summary = dict(campaign.summary or {}) summary["scheduler"] = scheduler_state campaign.summary = summary repo.update(campaign) return result # ── durable scheduler loop ────────────────────────────────────────────────── async def run_campaign_loop( campaign_id: str, cancel: asyncio.Event, *, tick_seconds: float = DEFAULT_TICK_SECONDS ) -> None: """Drive one campaign to completion, ticking on real wall-clock time. The loop holds no authority and no decision logic: every tick it reloads the campaign, asks the pure ``decide_tick`` what to do, and only performs I/O. It exits when the window finishes, the campaign leaves RUNNING (e.g. it was cancelled), or the cooperative cancel event fires. """ session = get_session() try: while not cancel.is_set(): repo = CampaignRepository(session) campaign = repo.get(campaign_id) if not campaign: return now = utc_now() decision = decide_tick( now=now, started_at=campaign.started_at, status=campaign.status, window_seconds=campaign.window_seconds, time_scale=campaign.time_scale, plan=campaign.plan, spawned_indices=_spawned_indices(campaign), ) if decision.action is TickAction.STOP: return elapsed = elapsed_seconds(now=now, started_at=campaign.started_at) await advance_campaign( campaign_id=campaign_id, elapsed_seconds=elapsed, session=session, cancel_event=cancel, ) if decision.action is TickAction.COMPLETE: current = repo.get(campaign_id) # Cancel-race guard: only complete if still RUNNING (pure rule). if current and resolve_finalize(current.status) is TickAction.COMPLETE: current.status = CampaignStatus.COMPLETED current.completed_at = utc_now() repo.update(current) return try: await asyncio.wait_for(cancel.wait(), timeout=tick_seconds) except asyncio.TimeoutError: pass except Exception as exc: # a crashing loop must not take down the app _logger.warning("活动调度循环 %s 异常退出: %s", campaign_id, exc) finally: session.close() def start_campaign(campaign_id: str, session: Session, *, tick_seconds: float = DEFAULT_TICK_SECONDS) -> Optional[asyncio.Task]: """Move a campaign into RUNNING (stamping ``started_at`` on first start) and launch its loop. Reused for both create-then-start and restart recovery: a PLANNED campaign gets a fresh ``started_at``; an already-RUNNING one keeps its original window start so recovery resumes at the correct offset. """ repo = CampaignRepository(session) campaign = repo.get(campaign_id) if not campaign or campaign.status in (CampaignStatus.COMPLETED, CampaignStatus.CANCELLED, CampaignStatus.FAILED): return None if campaign.status == CampaignStatus.PLANNED: campaign.status = CampaignStatus.RUNNING campaign.started_at = utc_now() repo.update(campaign) return campaign_registry.launch( campaign_id, lambda cancel: run_campaign_loop(campaign_id, cancel, tick_seconds=tick_seconds), ) def request_cancel(campaign_id: str) -> None: """Signal the loop (if live) to stop spawning and exit promptly.""" campaign_registry.cancel(campaign_id) def resume_running_campaigns(session: Session, *, tick_seconds: float = DEFAULT_TICK_SECONDS) -> int: """On startup, relaunch a loop for every campaign left in RUNNING.""" resumed = 0 for campaign in CampaignRepository(session).list_all(): if campaign.status == CampaignStatus.RUNNING and campaign.id: start_campaign(campaign.id, session, tick_seconds=tick_seconds) resumed += 1 return resumed async def shutdown_all() -> None: """Gracefully stop all live loops (Web app shutdown).""" await campaign_registry.shutdown_all()