"""API routes for evaluation campaigns (评估活动). Creating a campaign starts its durable scheduler loop (``campaign_runner``), which spawns child Runs across the (optionally compressed) service-cycle window until it finishes. Progress is authoritative in the DB, so detail queries report the live window position and spawned/completed Run counts, and a campaign can be cancelled mid-flight. """ from fastapi import APIRouter, Body, Depends, HTTPException, Response from pydantic import BaseModel, Field from sqlmodel import Session from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign from agenteval.evaluation.comparison import ( build_campaign_report_dict, build_comparison_payload, compute_metric_diff, resolve_auto_baseline, start_campaign_comparison, ) from agenteval.evaluation.report import ( build_campaign_timeline, generate_campaign_report, summarize_campaign_progress, ) from agenteval.evaluation.report_render import render_campaign_markdown from agenteval.exploration.summary import summarize_exploration from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds from agenteval.storage.db import iso_utc, utc_now from agenteval.storage.model_config_repository import ModelConfigRepository from agenteval.storage.repository import ( CampaignAnalysisRepository, CampaignPeriodComparisonRepository, CampaignRepository, ExplorationSessionRepository, RunRepository, ScenarioRepository, TargetRepository, ) from agenteval.web.deps import get_db router = APIRouter() class CreateCampaignRequest(BaseModel): name: str target_id: str window_seconds: int = Field(gt=0) time_scale: float = Field(default=1.0, gt=0) plan: list[CampaignPlanEntry] = Field(min_length=1) analysis_model_config_id: str | None = None exploration_seeds: ExplorationSeeds | None = None exploration_budget: ExplorationBudgetConfig | None = None @router.get("") async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]: repo = CampaignRepository(session) run_repo = RunRepository(session) result = [] for campaign in repo.list_all(): data = campaign.model_dump() runs = run_repo.list_by_campaign(campaign.id) data["progress"] = summarize_campaign_progress(campaign, runs) result.append(data) return result @router.post("") async def create_campaign( request: CreateCampaignRequest, session: Session = Depends(get_db), ) -> dict: if not TargetRepository(session).get(request.target_id): raise HTTPException(status_code=404, detail="target not found") scenario_repo = ScenarioRepository(session) for entry in request.plan: if not scenario_repo.get(entry.scenario_id): raise HTTPException( status_code=404, detail=f"scenario not found: {entry.scenario_id}", ) if request.analysis_model_config_id is not None: if not ModelConfigRepository(session).get(request.analysis_model_config_id): raise HTTPException(status_code=400, detail="analysis model config not found") seeds = request.exploration_seeds if seeds is not None and not seeds.personas and not seeds.goals: seeds = None # 种子留空 = 该活动不参与探索 campaign = Campaign( name=request.name, target_id=request.target_id, window_seconds=request.window_seconds, time_scale=request.time_scale, plan=request.plan, analysis_model_config_id=request.analysis_model_config_id, exploration_seeds=seeds, exploration_budget=request.exploration_budget, ) repo = CampaignRepository(session) campaign = repo.create(campaign) # Kick off the durable loop; it moves the campaign into RUNNING. start_campaign(campaign.id, session) return (repo.get(campaign.id) or campaign).model_dump() @router.post("/{campaign_id}/cancel") async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict: repo = CampaignRepository(session) campaign = repo.get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") if campaign.status not in (CampaignStatus.PLANNED, CampaignStatus.RUNNING): raise HTTPException(status_code=400, detail="campaign is not in a cancellable state") campaign.status = CampaignStatus.CANCELLED campaign.completed_at = utc_now() repo.update(campaign) request_cancel(campaign_id) return campaign.model_dump() @router.get("/{campaign_id}/report") async def get_campaign_report(campaign_id: str, session: Session = Depends(get_db)) -> dict: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") runs = RunRepository(session).list_by_campaign(campaign_id) scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()} report = generate_campaign_report(campaign, runs, scenario_names=scenario_names) exploration = summarize_exploration(ExplorationSessionRepository(session).list_by_campaign(campaign_id)) if exploration is not None: report["exploration"] = exploration return report @router.get("/{campaign_id}/report/markdown") async def get_campaign_report_markdown(campaign_id: str, session: Session = Depends(get_db)) -> Response: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") runs = RunRepository(session).list_by_campaign(campaign_id) scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()} analysis_row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id) analysis = analysis_row.get_result() if analysis_row and analysis_row.status == "completed" else None target = TargetRepository(session).get(campaign.target_id) target_name = target.name if target else None comparison = None cmp_row = CampaignPeriodComparisonRepository(session).get_by_campaign(campaign_id) if cmp_row is not None and cmp_row.status == "completed" and cmp_row.get_result(): baseline = CampaignRepository(session).get(cmp_row.baseline_campaign_id) metric_diff = ( compute_metric_diff( build_campaign_report_dict(baseline, session), build_campaign_report_dict(campaign, session), ) if baseline is not None else None ) model_cfg = ModelConfigRepository(session).get(cmp_row.model_config_id) if cmp_row.model_config_id else None model_label = f"{model_cfg.name}({model_cfg.model_name})" if model_cfg and model_cfg.model_name else ( model_cfg.name if model_cfg else None ) comparison = { "result": cmp_row.get_result(), "baseline_name": baseline.name if baseline else None, "baseline_completed_at": iso_utc(baseline.completed_at) if baseline else None, "model_name": model_label, "updated_at": iso_utc(cmp_row.updated_at), "metric_diff": metric_diff, } md = render_campaign_markdown( generate_campaign_report(campaign, runs, scenario_names=scenario_names), analysis=analysis, comparison=comparison, exploration=summarize_exploration(ExplorationSessionRepository(session).list_by_campaign(campaign_id)), target_name=target_name, scenario_names=scenario_names, ) return Response( content=md, media_type="text/markdown; charset=utf-8", headers={"Content-Disposition": f'attachment; filename="campaign-report-{campaign_id}.md"'}, ) @router.get("/{campaign_id}/timeline") async def get_campaign_timeline(campaign_id: str, session: Session = Depends(get_db)) -> dict: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") runs = RunRepository(session).list_by_campaign(campaign_id) scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()} return {"entries": build_campaign_timeline(campaign, runs, scenario_names=scenario_names)} @router.get("/{campaign_id}/analysis") async def get_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict: if not CampaignRepository(session).get(campaign_id): raise HTTPException(status_code=404, detail="campaign not found") row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id) if row is None: return {"status": "none"} return { "status": row.status, "result": row.get_result(), "error": row.error, "model_config_id": row.model_config_id, "triggered_by": row.triggered_by, "updated_at": iso_utc(row.updated_at), } @router.post("/{campaign_id}/analysis") async def trigger_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING): raise HTTPException(status_code=400, detail="活动完成后才能生成智能分析") if resolve_analysis_model(campaign, session) is None: raise HTTPException( status_code=400, detail="未配置分析模型:请在模型配置中心将某个 chat 配置设为「分析默认」,或为该活动指定分析模型", ) start_campaign_analysis(campaign_id, triggered_by="manual") return {"status": "generating"} @router.get("/{campaign_id}/comparison") async def get_campaign_comparison(campaign_id: str, session: Session = Depends(get_db)) -> dict: """周期对比(v0.8):自动基线配对 + 叙述行状态 + 机械 diff(现算,不存储)。 ``metric_diff`` 对应「生效基线」:已有对比行时为该行记录的基线, 否则为自动基线;两者都没有时为 null。 """ campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") payload = build_comparison_payload(campaign, session) row = CampaignPeriodComparisonRepository(session).get_by_campaign(campaign_id) if row is None: return {"status": "none", "comparison": None, **payload} baseline = CampaignRepository(session).get(row.baseline_campaign_id) metric_diff = ( compute_metric_diff( build_campaign_report_dict(baseline, session), build_campaign_report_dict(campaign, session), ) if baseline is not None else None ) comparison = { "baseline_campaign_id": row.baseline_campaign_id, "baseline": ( { "id": baseline.id, "name": baseline.name, "completed_at": iso_utc(baseline.completed_at), } if baseline else None ), "result": row.get_result(), "error": row.error, "model_config_id": row.model_config_id, "triggered_by": row.triggered_by, "updated_at": iso_utc(row.updated_at), } return { "status": row.status, "comparison": comparison, "auto_baseline": payload["auto_baseline"], "metric_diff": metric_diff, } class GenerateComparisonRequest(BaseModel): baseline_campaign_id: str | None = None @router.post("/{campaign_id}/comparison") async def trigger_campaign_comparison( campaign_id: str, request: GenerateComparisonRequest = Body(default=None), session: Session = Depends(get_db), ) -> dict: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING): raise HTTPException(status_code=400, detail="活动完成后才能生成周期对比") analyses = CampaignAnalysisRepository(session) current_analysis = analyses.get_by_campaign(campaign_id) if current_analysis is None or current_analysis.status != "completed": raise HTTPException(status_code=400, detail="请先生成本期活动的智能分析") if resolve_analysis_model(campaign, session) is None: raise HTTPException( status_code=400, detail="未配置分析模型:请在模型配置中心将某个 chat 配置设为「分析默认」,或为该活动指定分析模型", ) explicit_baseline_id = request.baseline_campaign_id if request else None if explicit_baseline_id: baseline = CampaignRepository(session).get(explicit_baseline_id) if baseline is None: raise HTTPException(status_code=400, detail=f"基线活动不存在: {explicit_baseline_id}") baseline_analysis = analyses.get_by_campaign(explicit_baseline_id) if baseline_analysis is None or baseline_analysis.status != "completed": raise HTTPException(status_code=400, detail="基线活动没有已完成的智能分析,请先生成基线活动的分析") else: baseline = resolve_auto_baseline(campaign, session) if baseline is None: raise HTTPException( status_code=400, detail="未找到自动基线:历史活动中没有同计划指纹且已完成分析的活动,可手动选择基线活动", ) start_campaign_comparison(campaign_id, triggered_by="manual", baseline_campaign_id=baseline.id) return {"status": "generating"} @router.get("/{campaign_id}") async def get_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict: campaign = CampaignRepository(session).get(campaign_id) if not campaign: raise HTTPException(status_code=404, detail="campaign not found") runs = RunRepository(session).list_by_campaign(campaign_id) data = campaign.model_dump() data["progress"] = campaign_progress(campaign, runs) return data