344 lines
14 KiB
Python
344 lines
14 KiB
Python
"""API routes for evaluation campaigns (评估活动).
|
||
|
||
Creating a campaign starts its durable scheduler loop (``campaign_runner``),
|
||
which spawns child Runs across the (optionally compressed) service-cycle window
|
||
until it finishes. Progress is authoritative in the DB, so detail queries report
|
||
the live window position and spawned/completed Run counts, and a campaign can be
|
||
cancelled mid-flight.
|
||
"""
|
||
|
||
from fastapi import APIRouter, Body, Depends, HTTPException, Response
|
||
from pydantic import BaseModel, Field
|
||
from sqlmodel import Session
|
||
|
||
from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis
|
||
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
||
from agenteval.evaluation.comparison import (
|
||
build_campaign_report_dict,
|
||
build_comparison_payload,
|
||
compute_metric_diff,
|
||
resolve_auto_baseline,
|
||
start_campaign_comparison,
|
||
)
|
||
from agenteval.evaluation.report import (
|
||
build_campaign_timeline,
|
||
generate_campaign_report,
|
||
summarize_campaign_progress,
|
||
)
|
||
from agenteval.evaluation.report_render import render_campaign_markdown
|
||
from agenteval.exploration.settlement import settle_campaign_sessions
|
||
from agenteval.exploration.summary import summarize_exploration
|
||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
|
||
from agenteval.storage.db import iso_utc, utc_now
|
||
from agenteval.storage.model_config_repository import ModelConfigRepository
|
||
from agenteval.storage.repository import (
|
||
CampaignAnalysisRepository,
|
||
CampaignPeriodComparisonRepository,
|
||
CampaignRepository,
|
||
ExplorationSessionRepository,
|
||
RunRepository,
|
||
ScenarioRepository,
|
||
TargetRepository,
|
||
)
|
||
from agenteval.web.deps import get_db
|
||
|
||
router = APIRouter()
|
||
|
||
|
||
class CreateCampaignRequest(BaseModel):
|
||
name: str
|
||
target_id: str
|
||
window_seconds: int = Field(gt=0)
|
||
time_scale: float = Field(default=1.0, gt=0)
|
||
plan: list[CampaignPlanEntry] = Field(min_length=1)
|
||
analysis_model_config_id: str | None = None
|
||
exploration_seeds: ExplorationSeeds | None = None
|
||
exploration_budget: ExplorationBudgetConfig | None = None
|
||
|
||
|
||
@router.get("")
|
||
async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]:
|
||
repo = CampaignRepository(session)
|
||
run_repo = RunRepository(session)
|
||
result = []
|
||
for campaign in repo.list_all():
|
||
data = campaign.model_dump()
|
||
runs = run_repo.list_by_campaign(campaign.id)
|
||
data["progress"] = summarize_campaign_progress(campaign, runs)
|
||
result.append(data)
|
||
return result
|
||
|
||
|
||
@router.post("")
|
||
async def create_campaign(
|
||
request: CreateCampaignRequest,
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
if not TargetRepository(session).get(request.target_id):
|
||
raise HTTPException(status_code=404, detail="target not found")
|
||
|
||
scenario_repo = ScenarioRepository(session)
|
||
for entry in request.plan:
|
||
if not scenario_repo.get(entry.scenario_id):
|
||
raise HTTPException(
|
||
status_code=404,
|
||
detail=f"scenario not found: {entry.scenario_id}",
|
||
)
|
||
|
||
if request.analysis_model_config_id is not None:
|
||
if not ModelConfigRepository(session).get(request.analysis_model_config_id):
|
||
raise HTTPException(status_code=400, detail="analysis model config not found")
|
||
|
||
seeds = request.exploration_seeds
|
||
if seeds is not None and not seeds.personas and not seeds.goals:
|
||
seeds = None # 种子留空 = 该活动不参与探索
|
||
|
||
campaign = Campaign(
|
||
name=request.name,
|
||
target_id=request.target_id,
|
||
window_seconds=request.window_seconds,
|
||
time_scale=request.time_scale,
|
||
plan=request.plan,
|
||
analysis_model_config_id=request.analysis_model_config_id,
|
||
exploration_seeds=seeds,
|
||
exploration_budget=request.exploration_budget,
|
||
)
|
||
repo = CampaignRepository(session)
|
||
campaign = repo.create(campaign)
|
||
# Kick off the durable loop; it moves the campaign into RUNNING.
|
||
start_campaign(campaign.id, session)
|
||
return (repo.get(campaign.id) or campaign).model_dump()
|
||
|
||
|
||
@router.post("/{campaign_id}/cancel")
|
||
async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
repo = CampaignRepository(session)
|
||
campaign = repo.get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
if campaign.status not in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||
raise HTTPException(status_code=400, detail="campaign is not in a cancellable state")
|
||
|
||
campaign.status = CampaignStatus.CANCELLED
|
||
campaign.completed_at = utc_now()
|
||
repo.update(campaign)
|
||
request_cancel(campaign_id)
|
||
settle_campaign_sessions(campaign_id, session)
|
||
return campaign.model_dump()
|
||
|
||
|
||
@router.get("/{campaign_id}/report")
|
||
async def get_campaign_report(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
||
report = generate_campaign_report(campaign, runs, scenario_names=scenario_names)
|
||
exploration = summarize_exploration(ExplorationSessionRepository(session).list_by_campaign(campaign_id))
|
||
if exploration is not None:
|
||
report["exploration"] = exploration
|
||
return report
|
||
|
||
|
||
@router.get("/{campaign_id}/report/markdown")
|
||
async def get_campaign_report_markdown(campaign_id: str, session: Session = Depends(get_db)) -> Response:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
||
analysis_row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id)
|
||
analysis = analysis_row.get_result() if analysis_row and analysis_row.status == "completed" else None
|
||
target = TargetRepository(session).get(campaign.target_id)
|
||
target_name = target.name if target else None
|
||
|
||
comparison = None
|
||
cmp_row = CampaignPeriodComparisonRepository(session).get_by_campaign(campaign_id)
|
||
if cmp_row is not None and cmp_row.status == "completed" and cmp_row.get_result():
|
||
baseline = CampaignRepository(session).get(cmp_row.baseline_campaign_id)
|
||
metric_diff = (
|
||
compute_metric_diff(
|
||
build_campaign_report_dict(baseline, session),
|
||
build_campaign_report_dict(campaign, session),
|
||
)
|
||
if baseline is not None
|
||
else None
|
||
)
|
||
model_cfg = ModelConfigRepository(session).get(cmp_row.model_config_id) if cmp_row.model_config_id else None
|
||
model_label = f"{model_cfg.name}（{model_cfg.model_name}）" if model_cfg and model_cfg.model_name else (
|
||
model_cfg.name if model_cfg else None
|
||
)
|
||
comparison = {
|
||
"result": cmp_row.get_result(),
|
||
"baseline_name": baseline.name if baseline else None,
|
||
"baseline_completed_at": iso_utc(baseline.completed_at) if baseline else None,
|
||
"model_name": model_label,
|
||
"updated_at": iso_utc(cmp_row.updated_at),
|
||
"metric_diff": metric_diff,
|
||
}
|
||
|
||
md = render_campaign_markdown(
|
||
generate_campaign_report(campaign, runs, scenario_names=scenario_names),
|
||
analysis=analysis,
|
||
comparison=comparison,
|
||
exploration=summarize_exploration(ExplorationSessionRepository(session).list_by_campaign(campaign_id)),
|
||
target_name=target_name,
|
||
scenario_names=scenario_names,
|
||
)
|
||
return Response(
|
||
content=md,
|
||
media_type="text/markdown; charset=utf-8",
|
||
headers={"Content-Disposition": f'attachment; filename="campaign-report-{campaign_id}.md"'},
|
||
)
|
||
|
||
|
||
@router.get("/{campaign_id}/timeline")
|
||
async def get_campaign_timeline(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
||
return {"entries": build_campaign_timeline(campaign, runs, scenario_names=scenario_names)}
|
||
|
||
|
||
@router.get("/{campaign_id}/analysis")
|
||
async def get_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
if not CampaignRepository(session).get(campaign_id):
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id)
|
||
if row is None:
|
||
return {"status": "none"}
|
||
return {
|
||
"status": row.status,
|
||
"result": row.get_result(),
|
||
"error": row.error,
|
||
"model_config_id": row.model_config_id,
|
||
"triggered_by": row.triggered_by,
|
||
"updated_at": iso_utc(row.updated_at),
|
||
}
|
||
|
||
|
||
@router.post("/{campaign_id}/analysis")
|
||
async def trigger_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||
raise HTTPException(status_code=400, detail="活动完成后才能生成智能分析")
|
||
if resolve_analysis_model(campaign, session) is None:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」，或为该活动指定分析模型",
|
||
)
|
||
start_campaign_analysis(campaign_id, triggered_by="manual")
|
||
return {"status": "generating"}
|
||
|
||
|
||
@router.get("/{campaign_id}/comparison")
|
||
async def get_campaign_comparison(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
"""周期对比（v0.8）：自动基线配对 + 叙述行状态 + 机械 diff（现算，不存储）。
|
||
|
||
``metric_diff`` 对应「生效基线」：已有对比行时为该行记录的基线，
|
||
否则为自动基线；两者都没有时为 null。
|
||
"""
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
|
||
payload = build_comparison_payload(campaign, session)
|
||
row = CampaignPeriodComparisonRepository(session).get_by_campaign(campaign_id)
|
||
if row is None:
|
||
return {"status": "none", "comparison": None, **payload}
|
||
|
||
baseline = CampaignRepository(session).get(row.baseline_campaign_id)
|
||
metric_diff = (
|
||
compute_metric_diff(
|
||
build_campaign_report_dict(baseline, session),
|
||
build_campaign_report_dict(campaign, session),
|
||
)
|
||
if baseline is not None
|
||
else None
|
||
)
|
||
comparison = {
|
||
"baseline_campaign_id": row.baseline_campaign_id,
|
||
"baseline": (
|
||
{
|
||
"id": baseline.id,
|
||
"name": baseline.name,
|
||
"completed_at": iso_utc(baseline.completed_at),
|
||
}
|
||
if baseline
|
||
else None
|
||
),
|
||
"result": row.get_result(),
|
||
"error": row.error,
|
||
"model_config_id": row.model_config_id,
|
||
"triggered_by": row.triggered_by,
|
||
"updated_at": iso_utc(row.updated_at),
|
||
}
|
||
return {
|
||
"status": row.status,
|
||
"comparison": comparison,
|
||
"auto_baseline": payload["auto_baseline"],
|
||
"metric_diff": metric_diff,
|
||
}
|
||
|
||
|
||
class GenerateComparisonRequest(BaseModel):
|
||
baseline_campaign_id: str | None = None
|
||
|
||
|
||
@router.post("/{campaign_id}/comparison")
|
||
async def trigger_campaign_comparison(
|
||
campaign_id: str,
|
||
request: GenerateComparisonRequest = Body(default=None),
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||
raise HTTPException(status_code=400, detail="活动完成后才能生成周期对比")
|
||
analyses = CampaignAnalysisRepository(session)
|
||
current_analysis = analyses.get_by_campaign(campaign_id)
|
||
if current_analysis is None or current_analysis.status != "completed":
|
||
raise HTTPException(status_code=400, detail="请先生成本期活动的智能分析")
|
||
if resolve_analysis_model(campaign, session) is None:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」，或为该活动指定分析模型",
|
||
)
|
||
|
||
explicit_baseline_id = request.baseline_campaign_id if request else None
|
||
if explicit_baseline_id:
|
||
baseline = CampaignRepository(session).get(explicit_baseline_id)
|
||
if baseline is None:
|
||
raise HTTPException(status_code=400, detail=f"基线活动不存在: {explicit_baseline_id}")
|
||
baseline_analysis = analyses.get_by_campaign(explicit_baseline_id)
|
||
if baseline_analysis is None or baseline_analysis.status != "completed":
|
||
raise HTTPException(status_code=400, detail="基线活动没有已完成的智能分析，请先生成基线活动的分析")
|
||
else:
|
||
baseline = resolve_auto_baseline(campaign, session)
|
||
if baseline is None:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="未找到自动基线：历史活动中没有同计划指纹且已完成分析的活动，可手动选择基线活动",
|
||
)
|
||
|
||
start_campaign_comparison(campaign_id, triggered_by="manual", baseline_campaign_id=baseline.id)
|
||
return {"status": "generating"}
|
||
|
||
|
||
@router.get("/{campaign_id}")
|
||
async def get_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
data = campaign.model_dump()
|
||
data["progress"] = campaign_progress(campaign, runs)
|
||
return data
|