53afb9b5d1
v0.9 ticket 02. Campaigns now carry an exploration seed set (seed personas × seed goals — the comparability unit for exploratory evaluation) and an optional budget override, stored as JSON columns isomorphic to plan. Empty seeds normalize to null, marking the campaign as opted out of exploration. resolve_budget merges per-field overrides into platform defaults; enforcement stays server-side. The create form gains seed lists and budget inputs (minutes → seconds), submitting null when left empty.
335 lines
14 KiB
Python
335 lines
14 KiB
Python
"""API routes for evaluation campaigns (评估活动).
|
||
|
||
Creating a campaign starts its durable scheduler loop (``campaign_runner``),
|
||
which spawns child Runs across the (optionally compressed) service-cycle window
|
||
until it finishes. Progress is authoritative in the DB, so detail queries report
|
||
the live window position and spawned/completed Run counts, and a campaign can be
|
||
cancelled mid-flight.
|
||
"""
|
||
|
||
from fastapi import APIRouter, Body, Depends, HTTPException, Response
|
||
from pydantic import BaseModel, Field
|
||
from sqlmodel import Session
|
||
|
||
from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis
|
||
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
||
from agenteval.evaluation.comparison import (
|
||
build_campaign_report_dict,
|
||
build_comparison_payload,
|
||
compute_metric_diff,
|
||
resolve_auto_baseline,
|
||
start_campaign_comparison,
|
||
)
|
||
from agenteval.evaluation.report import (
|
||
build_campaign_timeline,
|
||
generate_campaign_report,
|
||
summarize_campaign_progress,
|
||
)
|
||
from agenteval.evaluation.report_render import render_campaign_markdown
|
||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
|
||
from agenteval.storage.db import iso_utc, utc_now
|
||
from agenteval.storage.model_config_repository import ModelConfigRepository
|
||
from agenteval.storage.repository import (
|
||
CampaignAnalysisRepository,
|
||
CampaignPeriodComparisonRepository,
|
||
CampaignRepository,
|
||
RunRepository,
|
||
ScenarioRepository,
|
||
TargetRepository,
|
||
)
|
||
from agenteval.web.deps import get_db
|
||
|
||
router = APIRouter()
|
||
|
||
|
||
class CreateCampaignRequest(BaseModel):
|
||
name: str
|
||
target_id: str
|
||
window_seconds: int = Field(gt=0)
|
||
time_scale: float = Field(default=1.0, gt=0)
|
||
plan: list[CampaignPlanEntry] = Field(min_length=1)
|
||
analysis_model_config_id: str | None = None
|
||
exploration_seeds: ExplorationSeeds | None = None
|
||
exploration_budget: ExplorationBudgetConfig | None = None
|
||
|
||
|
||
@router.get("")
|
||
async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]:
|
||
repo = CampaignRepository(session)
|
||
run_repo = RunRepository(session)
|
||
result = []
|
||
for campaign in repo.list_all():
|
||
data = campaign.model_dump()
|
||
runs = run_repo.list_by_campaign(campaign.id)
|
||
data["progress"] = summarize_campaign_progress(campaign, runs)
|
||
result.append(data)
|
||
return result
|
||
|
||
|
||
@router.post("")
|
||
async def create_campaign(
|
||
request: CreateCampaignRequest,
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
if not TargetRepository(session).get(request.target_id):
|
||
raise HTTPException(status_code=404, detail="target not found")
|
||
|
||
scenario_repo = ScenarioRepository(session)
|
||
for entry in request.plan:
|
||
if not scenario_repo.get(entry.scenario_id):
|
||
raise HTTPException(
|
||
status_code=404,
|
||
detail=f"scenario not found: {entry.scenario_id}",
|
||
)
|
||
|
||
if request.analysis_model_config_id is not None:
|
||
if not ModelConfigRepository(session).get(request.analysis_model_config_id):
|
||
raise HTTPException(status_code=400, detail="analysis model config not found")
|
||
|
||
seeds = request.exploration_seeds
|
||
if seeds is not None and not seeds.personas and not seeds.goals:
|
||
seeds = None # 种子留空 = 该活动不参与探索
|
||
|
||
campaign = Campaign(
|
||
name=request.name,
|
||
target_id=request.target_id,
|
||
window_seconds=request.window_seconds,
|
||
time_scale=request.time_scale,
|
||
plan=request.plan,
|
||
analysis_model_config_id=request.analysis_model_config_id,
|
||
exploration_seeds=seeds,
|
||
exploration_budget=request.exploration_budget,
|
||
)
|
||
repo = CampaignRepository(session)
|
||
campaign = repo.create(campaign)
|
||
# Kick off the durable loop; it moves the campaign into RUNNING.
|
||
start_campaign(campaign.id, session)
|
||
return (repo.get(campaign.id) or campaign).model_dump()
|
||
|
||
|
||
@router.post("/{campaign_id}/cancel")
|
||
async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
repo = CampaignRepository(session)
|
||
campaign = repo.get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
if campaign.status not in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||
raise HTTPException(status_code=400, detail="campaign is not in a cancellable state")
|
||
|
||
campaign.status = CampaignStatus.CANCELLED
|
||
campaign.completed_at = utc_now()
|
||
repo.update(campaign)
|
||
request_cancel(campaign_id)
|
||
return campaign.model_dump()
|
||
|
||
|
||
@router.get("/{campaign_id}/report")
|
||
async def get_campaign_report(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
||
return generate_campaign_report(campaign, runs, scenario_names=scenario_names)
|
||
|
||
|
||
@router.get("/{campaign_id}/report/markdown")
|
||
async def get_campaign_report_markdown(campaign_id: str, session: Session = Depends(get_db)) -> Response:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
||
analysis_row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id)
|
||
analysis = analysis_row.get_result() if analysis_row and analysis_row.status == "completed" else None
|
||
target = TargetRepository(session).get(campaign.target_id)
|
||
target_name = target.name if target else None
|
||
|
||
comparison = None
|
||
cmp_row = CampaignPeriodComparisonRepository(session).get_by_campaign(campaign_id)
|
||
if cmp_row is not None and cmp_row.status == "completed" and cmp_row.get_result():
|
||
baseline = CampaignRepository(session).get(cmp_row.baseline_campaign_id)
|
||
metric_diff = (
|
||
compute_metric_diff(
|
||
build_campaign_report_dict(baseline, session),
|
||
build_campaign_report_dict(campaign, session),
|
||
)
|
||
if baseline is not None
|
||
else None
|
||
)
|
||
model_cfg = ModelConfigRepository(session).get(cmp_row.model_config_id) if cmp_row.model_config_id else None
|
||
model_label = f"{model_cfg.name}（{model_cfg.model_name}）" if model_cfg and model_cfg.model_name else (
|
||
model_cfg.name if model_cfg else None
|
||
)
|
||
comparison = {
|
||
"result": cmp_row.get_result(),
|
||
"baseline_name": baseline.name if baseline else None,
|
||
"baseline_completed_at": iso_utc(baseline.completed_at) if baseline else None,
|
||
"model_name": model_label,
|
||
"updated_at": iso_utc(cmp_row.updated_at),
|
||
"metric_diff": metric_diff,
|
||
}
|
||
|
||
md = render_campaign_markdown(
|
||
generate_campaign_report(campaign, runs, scenario_names=scenario_names),
|
||
analysis=analysis,
|
||
comparison=comparison,
|
||
target_name=target_name,
|
||
scenario_names=scenario_names,
|
||
)
|
||
return Response(
|
||
content=md,
|
||
media_type="text/markdown; charset=utf-8",
|
||
headers={"Content-Disposition": f'attachment; filename="campaign-report-{campaign_id}.md"'},
|
||
)
|
||
|
||
|
||
@router.get("/{campaign_id}/timeline")
|
||
async def get_campaign_timeline(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
||
return {"entries": build_campaign_timeline(campaign, runs, scenario_names=scenario_names)}
|
||
|
||
|
||
@router.get("/{campaign_id}/analysis")
|
||
async def get_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
if not CampaignRepository(session).get(campaign_id):
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id)
|
||
if row is None:
|
||
return {"status": "none"}
|
||
return {
|
||
"status": row.status,
|
||
"result": row.get_result(),
|
||
"error": row.error,
|
||
"model_config_id": row.model_config_id,
|
||
"triggered_by": row.triggered_by,
|
||
"updated_at": iso_utc(row.updated_at),
|
||
}
|
||
|
||
|
||
@router.post("/{campaign_id}/analysis")
|
||
async def trigger_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||
raise HTTPException(status_code=400, detail="活动完成后才能生成智能分析")
|
||
if resolve_analysis_model(campaign, session) is None:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」，或为该活动指定分析模型",
|
||
)
|
||
start_campaign_analysis(campaign_id, triggered_by="manual")
|
||
return {"status": "generating"}
|
||
|
||
|
||
@router.get("/{campaign_id}/comparison")
|
||
async def get_campaign_comparison(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
"""周期对比（v0.8）：自动基线配对 + 叙述行状态 + 机械 diff（现算，不存储）。
|
||
|
||
``metric_diff`` 对应「生效基线」：已有对比行时为该行记录的基线，
|
||
否则为自动基线；两者都没有时为 null。
|
||
"""
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
|
||
payload = build_comparison_payload(campaign, session)
|
||
row = CampaignPeriodComparisonRepository(session).get_by_campaign(campaign_id)
|
||
if row is None:
|
||
return {"status": "none", "comparison": None, **payload}
|
||
|
||
baseline = CampaignRepository(session).get(row.baseline_campaign_id)
|
||
metric_diff = (
|
||
compute_metric_diff(
|
||
build_campaign_report_dict(baseline, session),
|
||
build_campaign_report_dict(campaign, session),
|
||
)
|
||
if baseline is not None
|
||
else None
|
||
)
|
||
comparison = {
|
||
"baseline_campaign_id": row.baseline_campaign_id,
|
||
"baseline": (
|
||
{
|
||
"id": baseline.id,
|
||
"name": baseline.name,
|
||
"completed_at": iso_utc(baseline.completed_at),
|
||
}
|
||
if baseline
|
||
else None
|
||
),
|
||
"result": row.get_result(),
|
||
"error": row.error,
|
||
"model_config_id": row.model_config_id,
|
||
"triggered_by": row.triggered_by,
|
||
"updated_at": iso_utc(row.updated_at),
|
||
}
|
||
return {
|
||
"status": row.status,
|
||
"comparison": comparison,
|
||
"auto_baseline": payload["auto_baseline"],
|
||
"metric_diff": metric_diff,
|
||
}
|
||
|
||
|
||
class GenerateComparisonRequest(BaseModel):
|
||
baseline_campaign_id: str | None = None
|
||
|
||
|
||
@router.post("/{campaign_id}/comparison")
|
||
async def trigger_campaign_comparison(
|
||
campaign_id: str,
|
||
request: GenerateComparisonRequest = Body(default=None),
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||
raise HTTPException(status_code=400, detail="活动完成后才能生成周期对比")
|
||
analyses = CampaignAnalysisRepository(session)
|
||
current_analysis = analyses.get_by_campaign(campaign_id)
|
||
if current_analysis is None or current_analysis.status != "completed":
|
||
raise HTTPException(status_code=400, detail="请先生成本期活动的智能分析")
|
||
if resolve_analysis_model(campaign, session) is None:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」，或为该活动指定分析模型",
|
||
)
|
||
|
||
explicit_baseline_id = request.baseline_campaign_id if request else None
|
||
if explicit_baseline_id:
|
||
baseline = CampaignRepository(session).get(explicit_baseline_id)
|
||
if baseline is None:
|
||
raise HTTPException(status_code=400, detail=f"基线活动不存在: {explicit_baseline_id}")
|
||
baseline_analysis = analyses.get_by_campaign(explicit_baseline_id)
|
||
if baseline_analysis is None or baseline_analysis.status != "completed":
|
||
raise HTTPException(status_code=400, detail="基线活动没有已完成的智能分析，请先生成基线活动的分析")
|
||
else:
|
||
baseline = resolve_auto_baseline(campaign, session)
|
||
if baseline is None:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="未找到自动基线：历史活动中没有同计划指纹且已完成分析的活动，可手动选择基线活动",
|
||
)
|
||
|
||
start_campaign_comparison(campaign_id, triggered_by="manual", baseline_campaign_id=baseline.id)
|
||
return {"status": "generating"}
|
||
|
||
|
||
@router.get("/{campaign_id}")
|
||
async def get_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
data = campaign.model_dump()
|
||
data["progress"] = campaign_progress(campaign, runs)
|
||
return data
|