245 lines
9.6 KiB
Python
245 lines
9.6 KiB
Python
"""API routes for evaluation campaigns (评估活动).
|
||
|
||
Creating a campaign starts its durable scheduler loop (``campaign_runner``),
|
||
which spawns child Runs across the (optionally compressed) service-cycle window
|
||
until it finishes. Progress is authoritative in the DB, so detail queries report
|
||
the live window position and spawned/completed Run counts, and a campaign can be
|
||
cancelled mid-flight.
|
||
"""
|
||
|
||
from fastapi import APIRouter, Body, Depends, HTTPException, Response
|
||
from pydantic import BaseModel, Field
|
||
from sqlmodel import Session
|
||
|
||
from agenteval.evaluation.analysis import enqueue_campaign_analysis, resolve_analysis_model
|
||
from agenteval.evaluation.campaign_lifecycle import CampaignCreateError, CampaignLifecycleError
|
||
from agenteval.evaluation.campaign_lifecycle import cancel_campaign as cancel_campaign_lifecycle
|
||
from agenteval.evaluation.campaign_lifecycle import create_campaign as create_campaign_lifecycle
|
||
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
||
from agenteval.evaluation.comparison import (
|
||
ComparisonError,
|
||
load_comparison_view,
|
||
start_campaign_comparison,
|
||
validate_comparison_request,
|
||
)
|
||
from agenteval.evaluation.report import (
|
||
build_campaign_timeline,
|
||
load_campaign_report,
|
||
load_campaign_view,
|
||
summarize_campaign_progress,
|
||
)
|
||
from agenteval.evaluation.report_render import render_campaign_markdown
|
||
from agenteval.exploration.summary import summarize_campaign_exploration
|
||
from agenteval.models import CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
|
||
from agenteval.storage.db import iso_utc
|
||
from agenteval.storage.repository import (
|
||
CampaignAnalysisRepository,
|
||
CampaignRepository,
|
||
RunRepository,
|
||
ScenarioRepository,
|
||
TargetRepository,
|
||
)
|
||
from agenteval.web.deps import get_db
|
||
|
||
router = APIRouter()
|
||
|
||
|
||
class CreateCampaignRequest(BaseModel):
|
||
name: str
|
||
target_id: str
|
||
window_seconds: int = Field(gt=0)
|
||
time_scale: float = Field(default=1.0, gt=0)
|
||
plan: list[CampaignPlanEntry] = Field(min_length=1)
|
||
analysis_model_config_id: str | None = None
|
||
exploration_seeds: ExplorationSeeds | None = None
|
||
exploration_budget: ExplorationBudgetConfig | None = None
|
||
|
||
|
||
@router.get("")
|
||
async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]:
|
||
repo = CampaignRepository(session)
|
||
run_repo = RunRepository(session)
|
||
result = []
|
||
for campaign in repo.list_all():
|
||
data = campaign.model_dump()
|
||
runs = run_repo.list_by_campaign(campaign.id)
|
||
data["progress"] = summarize_campaign_progress(campaign, runs)
|
||
result.append(data)
|
||
return result
|
||
|
||
|
||
@router.post("")
|
||
async def create_campaign(
|
||
request: CreateCampaignRequest,
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
try:
|
||
campaign = create_campaign_lifecycle(
|
||
session,
|
||
name=request.name,
|
||
target_id=request.target_id,
|
||
window_seconds=request.window_seconds,
|
||
time_scale=request.time_scale,
|
||
plan=request.plan,
|
||
analysis_model_config_id=request.analysis_model_config_id,
|
||
exploration_seeds=request.exploration_seeds,
|
||
exploration_budget=request.exploration_budget,
|
||
launch=start_campaign,
|
||
)
|
||
except CampaignCreateError as exc:
|
||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||
return campaign.model_dump()
|
||
|
||
|
||
@router.post("/{campaign_id}/cancel")
|
||
async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
try:
|
||
campaign = cancel_campaign_lifecycle(session, campaign_id, stop=request_cancel)
|
||
except CampaignLifecycleError as exc:
|
||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||
return campaign.model_dump()
|
||
|
||
|
||
@router.get("/{campaign_id}/report")
|
||
async def get_campaign_report(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
report = load_campaign_report(session, campaign)
|
||
exploration = summarize_campaign_exploration(session, campaign_id)
|
||
if exploration is not None:
|
||
report["exploration"] = exploration
|
||
return report
|
||
|
||
|
||
@router.get("/{campaign_id}/report/markdown")
|
||
async def get_campaign_report_markdown(campaign_id: str, session: Session = Depends(get_db)) -> Response:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
view = load_campaign_view(session, campaign)
|
||
scenario_names = ScenarioRepository(session).name_map()
|
||
target = TargetRepository(session).get(campaign.target_id)
|
||
target_name = target.name if target else None
|
||
|
||
comparison = None
|
||
if view["comparison"] and view["comparison"].get("status") == "completed":
|
||
cmp_view = view["comparison"]
|
||
cmp = cmp_view.get("comparison") or {}
|
||
baseline = cmp.get("baseline") or {}
|
||
comparison = {
|
||
"result": cmp.get("result"),
|
||
"baseline_name": baseline.get("name"),
|
||
"baseline_completed_at": baseline.get("completed_at"),
|
||
"model_name": cmp.get("model_name"),
|
||
"updated_at": cmp.get("updated_at"),
|
||
"metric_diff": cmp_view.get("metric_diff"),
|
||
}
|
||
|
||
md = render_campaign_markdown(
|
||
view["report"],
|
||
analysis=view["analysis"],
|
||
comparison=comparison,
|
||
exploration=view["exploration"],
|
||
target_name=target_name,
|
||
scenario_names=scenario_names,
|
||
)
|
||
return Response(
|
||
content=md,
|
||
media_type="text/markdown; charset=utf-8",
|
||
headers={"Content-Disposition": f'attachment; filename="campaign-report-{campaign_id}.md"'},
|
||
)
|
||
|
||
|
||
@router.get("/{campaign_id}/timeline")
|
||
async def get_campaign_timeline(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
scenario_names = ScenarioRepository(session).name_map()
|
||
return {"entries": build_campaign_timeline(campaign, runs, scenario_names=scenario_names)}
|
||
|
||
|
||
@router.get("/{campaign_id}/analysis")
|
||
async def get_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
if not CampaignRepository(session).get(campaign_id):
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id)
|
||
if row is None:
|
||
return {"status": "none"}
|
||
return {
|
||
"status": row.status,
|
||
"result": row.get_result(),
|
||
"error": row.error,
|
||
"model_config_id": row.model_config_id,
|
||
"triggered_by": row.triggered_by,
|
||
"updated_at": iso_utc(row.updated_at),
|
||
}
|
||
|
||
|
||
@router.post("/{campaign_id}/analysis")
|
||
async def trigger_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||
raise HTTPException(status_code=400, detail="活动完成后才能生成智能分析")
|
||
if resolve_analysis_model(campaign, session) is None:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」，或为该活动指定分析模型",
|
||
)
|
||
enqueue_campaign_analysis(campaign_id, triggered_by="manual")
|
||
return {"status": "generating"}
|
||
|
||
|
||
@router.get("/{campaign_id}/comparison")
|
||
async def get_campaign_comparison(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
"""周期对比（v0.8）：自动基线配对 + 叙述行状态 + 机械 diff（现算，不存储）。
|
||
|
||
``metric_diff`` 对应「生效基线」：已有对比行时为该行记录的基线，
|
||
否则为自动基线；两者都没有时为 null。
|
||
"""
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
|
||
return load_comparison_view(session, campaign)
|
||
|
||
|
||
class GenerateComparisonRequest(BaseModel):
|
||
baseline_campaign_id: str | None = None
|
||
|
||
|
||
@router.post("/{campaign_id}/comparison")
|
||
async def trigger_campaign_comparison(
|
||
campaign_id: str,
|
||
request: GenerateComparisonRequest = Body(default=None),
|
||
session: Session = Depends(get_db),
|
||
) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
|
||
explicit_baseline_id = request.baseline_campaign_id if request else None
|
||
try:
|
||
baseline = validate_comparison_request(session, campaign, explicit_baseline_id=explicit_baseline_id)
|
||
except ComparisonError as exc:
|
||
raise HTTPException(status_code=400, detail=str(exc))
|
||
|
||
start_campaign_comparison(campaign_id, triggered_by="manual", baseline_campaign_id=baseline.id)
|
||
return {"status": "generating"}
|
||
|
||
|
||
@router.get("/{campaign_id}")
|
||
async def get_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
raise HTTPException(status_code=404, detail="campaign not found")
|
||
|
||
runs = RunRepository(session).list_by_campaign(campaign_id)
|
||
data = campaign.model_dump()
|
||
data["progress"] = campaign_progress(campaign, runs)
|
||
return data
|