2b6cab6cb2
周期对比读模型升位为单一出口（load_comparison_view），GET/POST/markdown 三处调用点统一走同一 view 投影，消除「取数三件套」重复。校验逻辑收敛到 validate_comparison_request，router 捕获映射 400，执行器捕获落 failed 行， 校验顺序权威不再漂移。 - 新增 load_comparison_view：无行返回 status=none + auto_baseline，有行 返回完整 comparison dict（含 model_name 标签）+ metric_diff - 新增 validate_comparison_request：活动终态 → 模型 → 基线 → 分析，违 规抛 ComparisonError - execute_campaign_comparison 内联校验替换为 validate_comparison_request 调用，catch ComparisonError 落 failed 行 - router 三处迁移：GET /comparison、POST /comparison、markdown 导出 - 删除 build_comparison_payload（已吸收进 load_comparison_view） - 8 个新测试覆盖读模型三态 + 校验五错
269 lines
11 KiB
Python
269 lines
11 KiB
Python
"""API routes for evaluation campaigns (评估活动).
|
||
|
||
Creating a campaign starts its durable scheduler loop (``campaign_runner``),
|
||
which spawns child Runs across the (optionally compressed) service-cycle window
|
||
until it finishes. Progress is authoritative in the DB, so detail queries report
|
||
the live window position and spawned/completed Run counts, and a campaign can be
|
||
cancelled mid-flight.
|
||
"""
|
||
|
||
from fastapi import APIRouter, Body, Depends, HTTPException, Response
|
||
from pydantic import BaseModel, Field
|
||
from sqlmodel import Session
|
||
|
||
from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis
|
||
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
||
from agenteval.evaluation.comparison import (
|
||
ComparisonError,
|
||
load_comparison_view,
|
||
start_campaign_comparison,
|
||
validate_comparison_request,
|
||
)
|
||
from agenteval.evaluation.report import (
|
||
build_campaign_timeline,
|
||
load_campaign_report,
|
||
summarize_campaign_progress,
|
||
)
|
||
from agenteval.evaluation.report_render import render_campaign_markdown
|
||
from agenteval.exploration.settlement import settle_campaign_sessions
|
||
from agenteval.exploration.summary import summarize_campaign_exploration
|
||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
|
||
from agenteval.storage.db import iso_utc, utc_now
|
||
from agenteval.storage.model_config_repository import ModelConfigRepository
|
||
from agenteval.storage.repository import (
|
||
CampaignAnalysisRepository,
|
||
CampaignRepository,
|
||
RunRepository,
|
||
ScenarioRepository,
|
||
TargetRepository,
|
||
)
|
||
from agenteval.web.deps import get_db
|
||
|
||
router = APIRouter()
|
||
|
||
|
||
class CreateCampaignRequest(BaseModel):
|
||
name: str
|
||
target_id: str
|
||
window_seconds: int = Field(gt=0)
|
||
time_scale: float = Field(default=1.0, gt=0)
|
||
plan: list[CampaignPlanEntry] = Field(min_length=1)
|
||
analysis_model_config_id: str | None = None
|
||
exploration_seeds: ExplorationSeeds | None = None
|
||
exploration_budget: ExplorationBudgetConfig | None = None
|
||
|
||
|
||
@router.get("")
|
||
async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]:
|
||
repo = CampaignRepository(session)
|
||
run_repo = RunRepository(session)
|
||
result = []
|
||
for campaign in repo.list_all():
|
||
data = campaign.model_dump()
|
||
runs = run_repo.list_by_campaign(campaign.id)
|
||
data["progress"] = summarize_campaign_progress(campaign, runs)
|
||
result.append(data)
|
||
return result
|
||
|
||
|
||
@router.post("")
|
||
async def create_campaign(
|
||
request: CreateCampaignRequest,
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
if not TargetRepository(session).get(request.target_id):
|
||
raise HTTPException(status_code=404, detail="target not found")
|
||
|
||
scenario_repo = ScenarioRepository(session)
|
||
for entry in request.plan:
|
||
if not scenario_repo.get(entry.scenario_id):
|
||
raise HTTPException(
|
||
status_code=404,
|
||
detail=f"scenario not found: {entry.scenario_id}",
|
||
)
|
||
|
||
if request.analysis_model_config_id is not None:
|
||
if not ModelConfigRepository(session).get(request.analysis_model_config_id):
|
||
raise HTTPException(status_code=400, detail="analysis model config not found")
|
||
|
||
seeds = request.exploration_seeds
|
||
if seeds is not None and not seeds.personas and not seeds.goals:
|
||
seeds = None # 种子留空 = 该活动不参与探索
|
||
|
||
campaign = Campaign(
|
||
name=request.name,
|
||
target_id=request.target_id,
|
||
window_seconds=request.window_seconds,
|
||
time_scale=request.time_scale,
|
||
plan=request.plan,
|
||
analysis_model_config_id=request.analysis_model_config_id,
|
||
exploration_seeds=seeds,
|
||
exploration_budget=request.exploration_budget,
|
||
)
|
||
repo = CampaignRepository(session)
|
||
campaign = repo.create(campaign)
|
||
# Kick off the durable loop; it moves the campaign into RUNNING.
|
||
start_campaign(campaign.id, session)
|
||
return (repo.get(campaign.id) or campaign).model_dump()
|
||
|
||
|
||
@router.post("/{campaign_id}/cancel")
|
||
async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
repo = CampaignRepository(session)
|
||
campaign = repo.get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
if campaign.status not in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||
raise HTTPException(status_code=400, detail="campaign is not in a cancellable state")
|
||
|
||
campaign = repo.mark_cancelled(campaign_id, utc_now()) or campaign
|
||
request_cancel(campaign_id)
|
||
settle_campaign_sessions(campaign_id, session)
|
||
return campaign.model_dump()
|
||
|
||
|
||
@router.get("/{campaign_id}/report")
|
||
async def get_campaign_report(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
report = load_campaign_report(session, campaign)
|
||
exploration = summarize_campaign_exploration(session, campaign_id)
|
||
if exploration is not None:
|
||
report["exploration"] = exploration
|
||
return report
|
||
|
||
|
||
@router.get("/{campaign_id}/report/markdown")
|
||
async def get_campaign_report_markdown(campaign_id: str, session: Session = Depends(get_db)) -> Response:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
report = load_campaign_report(session, campaign)
|
||
scenario_names = ScenarioRepository(session).name_map()
|
||
analysis_row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id)
|
||
analysis = analysis_row.get_result() if analysis_row and analysis_row.status == "completed" else None
|
||
target = TargetRepository(session).get(campaign.target_id)
|
||
target_name = target.name if target else None
|
||
|
||
comparison = None
|
||
view = load_comparison_view(session, campaign)
|
||
if view["status"] == "completed" and view["comparison"] and view["comparison"].get("result"):
|
||
cmp = view["comparison"]
|
||
baseline = cmp.get("baseline") or {}
|
||
comparison = {
|
||
"result": cmp["result"],
|
||
"baseline_name": baseline.get("name"),
|
||
"baseline_completed_at": baseline.get("completed_at"),
|
||
"model_name": cmp.get("model_name"),
|
||
"updated_at": cmp.get("updated_at"),
|
||
"metric_diff": view["metric_diff"],
|
||
}
|
||
|
||
md = render_campaign_markdown(
|
||
report,
|
||
analysis=analysis,
|
||
comparison=comparison,
|
||
exploration=summarize_campaign_exploration(session, campaign_id),
|
||
target_name=target_name,
|
||
scenario_names=scenario_names,
|
||
)
|
||
return Response(
|
||
content=md,
|
||
media_type="text/markdown; charset=utf-8",
|
||
headers={"Content-Disposition": f'attachment; filename="campaign-report-{campaign_id}.md"'},
|
||
)
|
||
|
||
|
||
@router.get("/{campaign_id}/timeline")
|
||
async def get_campaign_timeline(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
scenario_names = ScenarioRepository(session).name_map()
|
||
return {"entries": build_campaign_timeline(campaign, runs, scenario_names=scenario_names)}
|
||
|
||
|
||
@router.get("/{campaign_id}/analysis")
|
||
async def get_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
if not CampaignRepository(session).get(campaign_id):
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id)
|
||
if row is None:
|
||
return {"status": "none"}
|
||
return {
|
||
"status": row.status,
|
||
"result": row.get_result(),
|
||
"error": row.error,
|
||
"model_config_id": row.model_config_id,
|
||
"triggered_by": row.triggered_by,
|
||
"updated_at": iso_utc(row.updated_at),
|
||
}
|
||
|
||
|
||
@router.post("/{campaign_id}/analysis")
|
||
async def trigger_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||
raise HTTPException(status_code=400, detail="活动完成后才能生成智能分析")
|
||
if resolve_analysis_model(campaign, session) is None:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」，或为该活动指定分析模型",
|
||
)
|
||
start_campaign_analysis(campaign_id, triggered_by="manual")
|
||
return {"status": "generating"}
|
||
|
||
|
||
@router.get("/{campaign_id}/comparison")
|
||
async def get_campaign_comparison(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
"""周期对比（v0.8）：自动基线配对 + 叙述行状态 + 机械 diff（现算，不存储）。
|
||
|
||
``metric_diff`` 对应「生效基线」：已有对比行时为该行记录的基线，
|
||
否则为自动基线；两者都没有时为 null。
|
||
"""
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
|
||
return load_comparison_view(session, campaign)
|
||
|
||
|
||
class GenerateComparisonRequest(BaseModel):
|
||
baseline_campaign_id: str | None = None
|
||
|
||
|
||
@router.post("/{campaign_id}/comparison")
|
||
async def trigger_campaign_comparison(
|
||
campaign_id: str,
|
||
request: GenerateComparisonRequest = Body(default=None),
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
|
||
explicit_baseline_id = request.baseline_campaign_id if request else None
|
||
try:
|
||
baseline = validate_comparison_request(session, campaign, explicit_baseline_id=explicit_baseline_id)
|
||
except ComparisonError as exc:
|
||
raise HTTPException(status_code=400, detail=str(exc))
|
||
|
||
start_campaign_comparison(campaign_id, triggered_by="manual", baseline_campaign_id=baseline.id)
|
||
return {"status": "generating"}
|
||
|
||
|
||
@router.get("/{campaign_id}")
|
||
async def get_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
data = campaign.model_dump()
|
||
data["progress"] = campaign_progress(campaign, runs)
|
||
return data
|