"""API routes for evaluation campaigns (评估活动). Creating a campaign starts its durable scheduler loop (``campaign_runner``), which spawns child Runs across the (optionally compressed) service-cycle window until it finishes. Progress is authoritative in the DB, so detail queries report the live window position and spawned/completed Run counts, and a campaign can be cancelled mid-flight. """ from fastapi import APIRouter, Body, Depends, HTTPException, Response from pydantic import BaseModel, Field from sqlmodel import Session from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign from agenteval.evaluation.comparison import ( ComparisonError, load_comparison_view, start_campaign_comparison, validate_comparison_request, ) from agenteval.evaluation.report import ( build_campaign_timeline, load_campaign_report, load_campaign_view, summarize_campaign_progress, ) from agenteval.evaluation.report_render import render_campaign_markdown from agenteval.exploration.settlement import settle_campaign_sessions from agenteval.exploration.summary import summarize_campaign_exploration from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds from agenteval.storage.db import iso_utc, utc_now from agenteval.storage.model_config_repository import ModelConfigRepository from agenteval.storage.repository import ( CampaignAnalysisRepository, CampaignRepository, RunRepository, ScenarioRepository, TargetRepository, ) from agenteval.web.deps import get_db router = APIRouter() class CreateCampaignRequest(BaseModel): name: str target_id: str window_seconds: int = Field(gt=0) time_scale: float = Field(default=1.0, gt=0) plan: list[CampaignPlanEntry] = Field(min_length=1) analysis_model_config_id: str | None = None exploration_seeds: ExplorationSeeds | None = None exploration_budget: ExplorationBudgetConfig | None = None @router.get("") async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]: repo = CampaignRepository(session) run_repo = RunRepository(session) result = [] for campaign in repo.list_all(): data = campaign.model_dump() runs = run_repo.list_by_campaign(campaign.id) data["progress"] = summarize_campaign_progress(campaign, runs) result.append(data) return result @router.post("") async def create_campaign( request: CreateCampaignRequest, session: Session = Depends(get_db), ) -> dict: if not TargetRepository(session).get(request.target_id): raise HTTPException(status_code=404, detail="target not found") scenario_repo = ScenarioRepository(session) for entry in request.plan: if not scenario_repo.get(entry.scenario_id): raise HTTPException( status_code=404, detail=f"scenario not found: {entry.scenario_id}", ) if request.analysis_model_config_id is not None: if not ModelConfigRepository(session).get(request.analysis_model_config_id): raise HTTPException(status_code=400, detail="analysis model config not found") seeds = request.exploration_seeds if seeds is not None and not seeds.personas and not seeds.goals: seeds = None # 种子留空 = 该活动不参与探索 campaign = Campaign( name=request.name, target_id=request.target_id, window_seconds=request.window_seconds, time_scale=request.time_scale, plan=request.plan, analysis_model_config_id=request.analysis_model_config_id, exploration_seeds=seeds, exploration_budget=request.exploration_budget, ) repo = CampaignRepository(session) campaign = repo.create(campaign) # Kick off the durable loop; it moves the campaign into RUNNING. start_campaign(campaign.id, session) return (repo.get(campaign.id) or campaign).model_dump() @router.post("/{campaign_id}/cancel") async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict: repo = CampaignRepository(session) campaign = repo.get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") if campaign.status not in (CampaignStatus.PLANNED, CampaignStatus.RUNNING): raise HTTPException(status_code=400, detail="campaign is not in a cancellable state") campaign = repo.mark_cancelled(campaign_id, utc_now()) or campaign request_cancel(campaign_id) settle_campaign_sessions(campaign_id, session) return campaign.model_dump() @router.get("/{campaign_id}/report") async def get_campaign_report(campaign_id: str, session: Session = Depends(get_db)) -> dict: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") report = load_campaign_report(session, campaign) exploration = summarize_campaign_exploration(session, campaign_id) if exploration is not None: report["exploration"] = exploration return report @router.get("/{campaign_id}/report/markdown") async def get_campaign_report_markdown(campaign_id: str, session: Session = Depends(get_db)) -> Response: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") view = load_campaign_view(session, campaign) scenario_names = ScenarioRepository(session).name_map() target = TargetRepository(session).get(campaign.target_id) target_name = target.name if target else None comparison = None if view["comparison"] and view["comparison"].get("status") == "completed": cmp_view = view["comparison"] cmp = cmp_view.get("comparison") or {} baseline = cmp.get("baseline") or {} comparison = { "result": cmp.get("result"), "baseline_name": baseline.get("name"), "baseline_completed_at": baseline.get("completed_at"), "model_name": cmp.get("model_name"), "updated_at": cmp.get("updated_at"), "metric_diff": cmp_view.get("metric_diff"), } md = render_campaign_markdown( view["report"], analysis=view["analysis"], comparison=comparison, exploration=view["exploration"], target_name=target_name, scenario_names=scenario_names, ) return Response( content=md, media_type="text/markdown; charset=utf-8", headers={"Content-Disposition": f'attachment; filename="campaign-report-{campaign_id}.md"'}, ) @router.get("/{campaign_id}/timeline") async def get_campaign_timeline(campaign_id: str, session: Session = Depends(get_db)) -> dict: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") runs = RunRepository(session).list_by_campaign(campaign_id) scenario_names = ScenarioRepository(session).name_map() return {"entries": build_campaign_timeline(campaign, runs, scenario_names=scenario_names)} @router.get("/{campaign_id}/analysis") async def get_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict: if not CampaignRepository(session).get(campaign_id): raise HTTPException(status_code=404, detail="campaign not found") row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id) if row is None: return {"status": "none"} return { "status": row.status, "result": row.get_result(), "error": row.error, "model_config_id": row.model_config_id, "triggered_by": row.triggered_by, "updated_at": iso_utc(row.updated_at), } @router.post("/{campaign_id}/analysis") async def trigger_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING): raise HTTPException(status_code=400, detail="活动完成后才能生成智能分析") if resolve_analysis_model(campaign, session) is None: raise HTTPException( status_code=400, detail="未配置分析模型:请在模型配置中心将某个 chat 配置设为「分析默认」,或为该活动指定分析模型", ) start_campaign_analysis(campaign_id, triggered_by="manual") return {"status": "generating"} @router.get("/{campaign_id}/comparison") async def get_campaign_comparison(campaign_id: str, session: Session = Depends(get_db)) -> dict: """周期对比(v0.8):自动基线配对 + 叙述行状态 + 机械 diff(现算,不存储)。 ``metric_diff`` 对应「生效基线」:已有对比行时为该行记录的基线, 否则为自动基线;两者都没有时为 null。 """ campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") return load_comparison_view(session, campaign) class GenerateComparisonRequest(BaseModel): baseline_campaign_id: str | None = None @router.post("/{campaign_id}/comparison") async def trigger_campaign_comparison( campaign_id: str, request: GenerateComparisonRequest = Body(default=None), session: Session = Depends(get_db), ) -> dict: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") explicit_baseline_id = request.baseline_campaign_id if request else None try: baseline = validate_comparison_request(session, campaign, explicit_baseline_id=explicit_baseline_id) except ComparisonError as exc: raise HTTPException(status_code=400, detail=str(exc)) start_campaign_comparison(campaign_id, triggered_by="manual", baseline_campaign_id=baseline.id) return {"status": "generating"} @router.get("/{campaign_id}") async def get_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") runs = RunRepository(session).list_by_campaign(campaign_id) data = campaign.model_dump() data["progress"] = campaign_progress(campaign, runs) return data