"""Campaign runner — the side-effect shell around the pure scheduler. Given a campaign and a clock position (real elapsed seconds), it asks ``campaign_scheduler.decide_schedule`` what is due, then durably claims each plan occurrence before reusing the existing single-run execution path (``EvalEngine.run`` with an ``existing_run``). Child-Run identities are the primary idempotency authority; the legacy campaign summary remains a fallback for pre-identity Runs created before this migration. This module also hosts the durable scheduler loop: a thin async shell that, tick by tick, maps real wall-clock elapsed time (since the campaign's persisted ``started_at``) to a window offset and calls ``advance_campaign``. All authority lives in the DB (window start, spawned progress, status), so the loop can be torn down and rebuilt on restart without losing or duplicating work. """ import asyncio import logging from dataclasses import dataclass, field from typing import Optional from sqlmodel import Session from agenteval.evaluation.analysis import enqueue_campaign_analysis, resolve_analysis_model from agenteval.evaluation.campaign_lifecycle import complete_campaign from agenteval.evaluation.campaign_lifecycle import start_campaign as start_campaign_lifecycle from agenteval.evaluation.campaign_scheduler import ( TickAction, clock_offset, decide_schedule, decide_tick, elapsed_seconds, resolve_finalize, ) from agenteval.evaluation.engine import EvalEngine from agenteval.models import ( Campaign, CampaignStatus, CampaignSummary, EvalRun, RunStatus, RunTrigger, SchedulerState, ) from agenteval.storage.db import get_session, utc_now from agenteval.storage.repository import ( CampaignRepository, CampaignRunClaimStatus, RunRepository, ScenarioRepository, TargetRepository, ) from agenteval.task_registry import TaskRegistry # Real wall-clock seconds between scheduler ticks. time_scale compresses the # *window*, not the tick cadence — production 24h campaigns still tick slowly. DEFAULT_TICK_SECONDS = 1.0 _logger = logging.getLogger("agenteval") # Live loop tasks + cooperative cancel events, keyed by campaign id. Authority # is the DB; this registry only holds the in-process handles for the loop. campaign_registry = TaskRegistry() @dataclass class AdvanceResult: """Outcome of advancing a campaign's clock once.""" spawned_run_ids: list[str] = field(default_factory=list) finished: bool = False @dataclass class CampaignRecoveryResult: """Outcome of reconciling child Runs left by a previous process.""" resumed_run_ids: list[str] = field(default_factory=list) failed_run_ids: list[str] = field(default_factory=list) def _spawned_indices(campaign: Campaign, session: Optional[Session] = None) -> set[int]: """Return plan entries completed by durable identities or legacy progress. Once any durable occurrence exists for an entry, identities replace the legacy summary for that entry. This repairs summaries written by older runners after only part of a multi-occurrence entry was claimed. """ legacy = set(campaign.summary.scheduler.spawned_indices) if campaign.summary else set() if not campaign.id or session is None: return legacy occurrences: dict[int, set[int]] = {} for run in RunRepository(session).list_by_campaign(campaign.id): if run.campaign_plan_index is None or run.campaign_occurrence_index is None: continue occurrences.setdefault(run.campaign_plan_index, set()).add(run.campaign_occurrence_index) return { plan_index for plan_index, entry in enumerate(campaign.plan) if ( plan_index in occurrences and set(range(entry.count)).issubset(occurrences[plan_index]) ) or (plan_index not in occurrences and plan_index in legacy) } def current_window_offset(campaign: Campaign) -> float: """The campaign's live window position (seconds), clamped to the window. RUNNING campaigns derive it from the persisted ``started_at`` and ``time_scale``; not-yet-started campaigns report 0. """ if campaign.started_at is None: return 0.0 offset = clock_offset( elapsed_seconds=elapsed_seconds(now=utc_now(), started_at=campaign.started_at), time_scale=campaign.time_scale, ) return min(offset, float(campaign.window_seconds)) def campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict: """Live progress of a campaign, derived from its child Runs. ``completed_runs`` counts only COMPLETED child Runs; failures stay out of this field (pass_rate semantics are ADR-0002's concern, not this counter). """ return { "current_offset_seconds": current_window_offset(campaign), "spawned_runs": len(runs), "completed_runs": sum(1 for r in runs if r.status == RunStatus.COMPLETED), } async def _spawn_child_run( campaign: Campaign, scenario_id: str, *, plan_index: int, occurrence_index: int, session: Session, ) -> Optional[str]: """Claim one occurrence and drive its pending Run through the engine. A rejected claim means the Campaign stopped being runnable between the scheduler decision and this write; callers must stop spawning. An existing pending Run is resumed, while an existing running/completed/failed Run is returned without replaying external messages. """ scenario = ScenarioRepository(session).get(scenario_id) if not scenario: raise ValueError(f"campaign scenario missing: {scenario_id}") claim = RunRepository(session).claim_campaign_run( campaign_id=campaign.id or "", scenario_id=scenario_id, scenario_version=scenario.version or 1, plan_index=plan_index, occurrence_index=occurrence_index, ) if claim.status in (CampaignRunClaimStatus.NOT_FOUND, CampaignRunClaimStatus.CONFLICT): return None run = claim.run if run is None: raise RuntimeError(f"Campaign child claim returned no Run: {claim.status.value}") if run.status is RunStatus.PENDING: target = TargetRepository(session).get(campaign.target_id) if not target: raise ValueError(f"campaign target missing: {campaign.target_id}") engine = EvalEngine(target=target, scenario=scenario, triggered_by=RunTrigger.CAMPAIGN) await engine.run(existing_run=run) return run.id or "" async def advance_campaign( *, campaign_id: str, elapsed_seconds: float, session: Session, cancel_event: Optional[asyncio.Event] = None, ) -> Optional[AdvanceResult]: """Advance the campaign clock to ``elapsed_seconds`` and spawn due Runs. ``elapsed_seconds`` is real wall-clock time since the window started; it is mapped to a window offset via ``time_scale``. A due entry whose spawning raises is recorded but remains due until every occurrence has a durable child-Run identity. Returns ``None`` if the campaign does not exist. """ repo = CampaignRepository(session) campaign = repo.get(campaign_id) if not campaign: return None offset = clock_offset(elapsed_seconds=elapsed_seconds, time_scale=campaign.time_scale) spawned = _spawned_indices(campaign, session) decision = decide_schedule( plan=campaign.plan, window_seconds=campaign.window_seconds, clock_offset_seconds=offset, spawned_indices=spawned, ) result = AdvanceResult(finished=decision.finished) errors: list[dict] = list(campaign.summary.scheduler.errors) if campaign.summary else [] for due in decision.due: # Cancellation stops further spawning; runs already in flight finish. if cancel_event is not None and cancel_event.is_set(): break claim_rejected = False try: for occurrence_index in range(due.entry.count): run_id = await _spawn_child_run( campaign, due.entry.scenario_id, plan_index=due.index, occurrence_index=occurrence_index, session=session, ) if run_id is None: claim_rejected = True break result.spawned_run_ids.append(run_id) except Exception as exc: # one entry failing must not wedge the campaign errors.append({"entry_index": due.index, "error": str(exc)}) _logger.warning("活动 %s 计划条目 %d 派生失败(跳过): %s", campaign_id, due.index, exc) if claim_rejected: break spawned = _spawned_indices(campaign, session) # Persist progress per entry: a failure partway through a multi-entry # advance must never lose which entries already spawned, since restart # recovery reads this back from the DB. Mutate the existing summary so # any unknown top-level keys survive the read-modify-write. summary = campaign.summary or CampaignSummary() summary.scheduler = SchedulerState(spawned_indices=sorted(spawned), errors=errors) campaign.summary = summary repo.save_scheduler_state(campaign_id, summary) return result def _auto_start_analysis(campaign: Campaign, session: Session) -> None: """Enqueue 智能分析 when a 正式线 campaign completes. Accelerated campaigns, an unresolvable analysis model, and any enqueue failure all skip silently — the analysis is an enhancement and must never block or break campaign completion. """ if campaign.time_scale != 1 or not campaign.id: return try: if resolve_analysis_model(campaign, session) is None: return enqueue_campaign_analysis(campaign.id, triggered_by="auto") except Exception as exc: _logger.warning("活动 %s 自动分析触发失败(已跳过): %s", campaign.id, exc) async def reconcile_campaign_child_runs( campaign_id: str, session: Session, *, cancel_event: Optional[asyncio.Event] = None, ) -> Optional[CampaignRecoveryResult]: """Resume safe pending claims and fail running Runs without replaying. A pending claim has not crossed the Engine's send seam and can be resumed. A running Run may already have reached the target, so recovery records an explicit interruption instead of executing it again. """ campaign = CampaignRepository(session).get(campaign_id) if campaign is None: return None if campaign.status is not CampaignStatus.RUNNING: return CampaignRecoveryResult() run_repo = RunRepository(session) result = CampaignRecoveryResult( failed_run_ids=run_repo.mark_campaign_running_interrupted(campaign_id) ) for run in run_repo.list_pending_campaign_children(campaign_id): if cancel_event is not None and cancel_event.is_set(): break plan_index = run.campaign_plan_index occurrence_index = run.campaign_occurrence_index valid_identity = ( plan_index is not None and occurrence_index is not None and 0 <= plan_index < len(campaign.plan) and 0 <= occurrence_index < campaign.plan[plan_index].count and run.scenario_id == campaign.plan[plan_index].scenario_id ) if not valid_identity: failed = run_repo.mark_pending_campaign_child_interrupted(run.id or "") if failed and failed.id: result.failed_run_ids.append(failed.id) continue try: resumed_id = await _spawn_child_run( campaign, run.scenario_id, plan_index=plan_index, occurrence_index=occurrence_index, session=session, ) if resumed_id is None: break result.resumed_run_ids.append(resumed_id) except Exception as exc: fresh = run_repo.get(run.id or "") if fresh is not None and fresh.status is RunStatus.PENDING: failed = run_repo.mark_pending_campaign_child_interrupted(run.id or "") if failed and failed.id: result.failed_run_ids.append(failed.id) elif fresh is not None and fresh.status is RunStatus.FAILED and fresh.id: result.failed_run_ids.append(fresh.id) _logger.warning("活动 %s 子运行 %s 恢复失败: %s", campaign_id, run.id, exc) return result # ── durable scheduler loop ────────────────────────────────────────────────── async def run_campaign_loop( campaign_id: str, cancel: asyncio.Event, *, tick_seconds: float = DEFAULT_TICK_SECONDS ) -> None: """Drive one campaign to completion, ticking on real wall-clock time. The loop holds no authority and no decision logic: every tick it reloads the campaign, asks the pure ``decide_tick`` what to do, and only performs I/O. It exits when the window finishes, the campaign leaves RUNNING (e.g. it was cancelled), or the cooperative cancel event fires. """ session = get_session() try: await reconcile_campaign_child_runs(campaign_id, session, cancel_event=cancel) while not cancel.is_set(): repo = CampaignRepository(session) campaign = repo.get(campaign_id) if not campaign: return now = utc_now() decision = decide_tick( now=now, started_at=campaign.started_at, status=campaign.status, window_seconds=campaign.window_seconds, time_scale=campaign.time_scale, plan=campaign.plan, spawned_indices=_spawned_indices(campaign, session), ) if decision.action is TickAction.STOP: return elapsed = elapsed_seconds(now=now, started_at=campaign.started_at) await advance_campaign( campaign_id=campaign_id, elapsed_seconds=elapsed, session=session, cancel_event=cancel, ) if decision.action is TickAction.COMPLETE: current = repo.get(campaign_id) # The lifecycle CAS is the authoritative cancel-race guard. if current and resolve_finalize(current.status) is TickAction.COMPLETE: try: completed = complete_campaign(campaign_id=campaign_id, session=session) except Exception as exc: _logger.warning("活动 %s 完成迁移失败: %s", campaign_id, exc) else: _auto_start_analysis(completed, session) return try: await asyncio.wait_for(cancel.wait(), timeout=tick_seconds) except asyncio.TimeoutError: pass except Exception as exc: # a crashing loop must not take down the app _logger.warning("活动调度循环 %s 异常退出: %s", campaign_id, exc) finally: session.close() def start_campaign(campaign_id: str, session: Session, *, tick_seconds: float = DEFAULT_TICK_SECONDS) -> Optional[asyncio.Task]: """Move a campaign into RUNNING (stamping ``started_at`` on first start) and launch its loop. Reused for both create-then-start and restart recovery: a PLANNED campaign gets a fresh ``started_at``; an already-RUNNING one keeps its original window start so recovery resumes at the correct offset. """ campaign = CampaignRepository(session).get(campaign_id) if not campaign or campaign.status in (CampaignStatus.COMPLETED, CampaignStatus.CANCELLED, CampaignStatus.FAILED): return None started = start_campaign_lifecycle(session, campaign_id) if started is None: return None return campaign_registry.launch( campaign_id, lambda cancel: run_campaign_loop(campaign_id, cancel, tick_seconds=tick_seconds), ) def request_cancel(campaign_id: str) -> None: """Signal the loop (if live) to stop spawning and exit promptly.""" campaign_registry.cancel(campaign_id) def resume_running_campaigns(session: Session, *, tick_seconds: float = DEFAULT_TICK_SECONDS) -> int: """On startup, relaunch a loop for every campaign left in RUNNING.""" resumed = 0 for campaign in CampaignRepository(session).list_all(): if campaign.status == CampaignStatus.RUNNING and campaign.id: start_campaign(campaign.id, session, tick_seconds=tick_seconds) resumed += 1 return resumed async def shutdown_all() -> None: """Gracefully stop all live loops (Web app shutdown).""" await campaign_registry.shutdown_all()