5d04455664
The scheduler loop enqueues the analysis task when a realtime campaign completes; accelerated or cancelled campaigns and a missing analysis model skip silently. The campaign markdown export appends the analysis appendix (overall, problems, narratives, suggestions) when a completed analysis exists.
194 lines
7.8 KiB
Python
194 lines
7.8 KiB
Python
"""API routes for evaluation campaigns (评估活动).
|
|
|
|
Creating a campaign starts its durable scheduler loop (``campaign_runner``),
|
|
which spawns child Runs across the (optionally compressed) service-cycle window
|
|
until it finishes. Progress is authoritative in the DB, so detail queries report
|
|
the live window position and spawned/completed Run counts, and a campaign can be
|
|
cancelled mid-flight.
|
|
"""
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Response
|
|
from pydantic import BaseModel, Field
|
|
from sqlmodel import Session
|
|
|
|
from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis
|
|
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
|
from agenteval.evaluation.report import (
|
|
build_campaign_timeline,
|
|
generate_campaign_report,
|
|
summarize_campaign_progress,
|
|
)
|
|
from agenteval.evaluation.report_render import render_campaign_markdown
|
|
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
|
|
from agenteval.storage.db import iso_utc, utc_now
|
|
from agenteval.storage.model_config_repository import ModelConfigRepository
|
|
from agenteval.storage.repository import (
|
|
CampaignAnalysisRepository,
|
|
CampaignRepository,
|
|
RunRepository,
|
|
ScenarioRepository,
|
|
TargetRepository,
|
|
)
|
|
from agenteval.web.deps import get_db
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
class CreateCampaignRequest(BaseModel):
|
|
name: str
|
|
target_id: str
|
|
window_seconds: int = Field(gt=0)
|
|
time_scale: float = Field(default=1.0, gt=0)
|
|
plan: list[CampaignPlanEntry] = Field(min_length=1)
|
|
analysis_model_config_id: str | None = None
|
|
|
|
|
|
@router.get("")
|
|
async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]:
|
|
repo = CampaignRepository(session)
|
|
run_repo = RunRepository(session)
|
|
result = []
|
|
for campaign in repo.list_all():
|
|
data = campaign.model_dump()
|
|
runs = run_repo.list_by_campaign(campaign.id)
|
|
data["progress"] = summarize_campaign_progress(campaign, runs)
|
|
result.append(data)
|
|
return result
|
|
|
|
|
|
@router.post("")
|
|
async def create_campaign(
|
|
request: CreateCampaignRequest,
|
|
session: Session = Depends(get_db),
|
|
) -> dict:
|
|
if not TargetRepository(session).get(request.target_id):
|
|
raise HTTPException(status_code=404, detail="target not found")
|
|
|
|
scenario_repo = ScenarioRepository(session)
|
|
for entry in request.plan:
|
|
if not scenario_repo.get(entry.scenario_id):
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail=f"scenario not found: {entry.scenario_id}",
|
|
)
|
|
|
|
if request.analysis_model_config_id is not None:
|
|
if not ModelConfigRepository(session).get(request.analysis_model_config_id):
|
|
raise HTTPException(status_code=400, detail="analysis model config not found")
|
|
|
|
campaign = Campaign(
|
|
name=request.name,
|
|
target_id=request.target_id,
|
|
window_seconds=request.window_seconds,
|
|
time_scale=request.time_scale,
|
|
plan=request.plan,
|
|
analysis_model_config_id=request.analysis_model_config_id,
|
|
)
|
|
repo = CampaignRepository(session)
|
|
campaign = repo.create(campaign)
|
|
# Kick off the durable loop; it moves the campaign into RUNNING.
|
|
start_campaign(campaign.id, session)
|
|
return (repo.get(campaign.id) or campaign).model_dump()
|
|
|
|
|
|
@router.post("/{campaign_id}/cancel")
|
|
async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
|
repo = CampaignRepository(session)
|
|
campaign = repo.get(campaign_id)
|
|
if not campaign:
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
if campaign.status not in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
|
raise HTTPException(status_code=400, detail="campaign is not in a cancellable state")
|
|
|
|
campaign.status = CampaignStatus.CANCELLED
|
|
campaign.completed_at = utc_now()
|
|
repo.update(campaign)
|
|
request_cancel(campaign_id)
|
|
return campaign.model_dump()
|
|
|
|
|
|
@router.get("/{campaign_id}/report")
|
|
async def get_campaign_report(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
|
campaign = CampaignRepository(session).get(campaign_id)
|
|
if not campaign:
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
runs = RunRepository(session).list_by_campaign(campaign_id)
|
|
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
|
return generate_campaign_report(campaign, runs, scenario_names=scenario_names)
|
|
|
|
|
|
@router.get("/{campaign_id}/report/markdown")
|
|
async def get_campaign_report_markdown(campaign_id: str, session: Session = Depends(get_db)) -> Response:
|
|
campaign = CampaignRepository(session).get(campaign_id)
|
|
if not campaign:
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
runs = RunRepository(session).list_by_campaign(campaign_id)
|
|
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
|
analysis_row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id)
|
|
analysis = analysis_row.get_result() if analysis_row and analysis_row.status == "completed" else None
|
|
md = render_campaign_markdown(
|
|
generate_campaign_report(campaign, runs, scenario_names=scenario_names),
|
|
analysis=analysis,
|
|
scenario_names=scenario_names,
|
|
)
|
|
return Response(
|
|
content=md,
|
|
media_type="text/markdown; charset=utf-8",
|
|
headers={"Content-Disposition": f'attachment; filename="campaign-report-{campaign_id}.md"'},
|
|
)
|
|
|
|
|
|
@router.get("/{campaign_id}/timeline")
|
|
async def get_campaign_timeline(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
|
campaign = CampaignRepository(session).get(campaign_id)
|
|
if not campaign:
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
runs = RunRepository(session).list_by_campaign(campaign_id)
|
|
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
|
return {"entries": build_campaign_timeline(campaign, runs, scenario_names=scenario_names)}
|
|
|
|
|
|
@router.get("/{campaign_id}/analysis")
|
|
async def get_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
|
if not CampaignRepository(session).get(campaign_id):
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
row = CampaignAnalysisRepository(session).get_by_campaign(campaign_id)
|
|
if row is None:
|
|
return {"status": "none"}
|
|
return {
|
|
"status": row.status,
|
|
"result": row.get_result(),
|
|
"error": row.error,
|
|
"model_config_id": row.model_config_id,
|
|
"triggered_by": row.triggered_by,
|
|
"updated_at": iso_utc(row.updated_at),
|
|
}
|
|
|
|
|
|
@router.post("/{campaign_id}/analysis")
|
|
async def trigger_campaign_analysis(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
|
campaign = CampaignRepository(session).get(campaign_id)
|
|
if not campaign:
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
|
raise HTTPException(status_code=400, detail="活动完成后才能生成智能分析")
|
|
if resolve_analysis_model(campaign, session) is None:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」，或为该活动指定分析模型",
|
|
)
|
|
start_campaign_analysis(campaign_id, triggered_by="manual")
|
|
return {"status": "generating"}
|
|
|
|
|
|
@router.get("/{campaign_id}")
|
|
async def get_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
|
campaign = CampaignRepository(session).get(campaign_id)
|
|
if not campaign:
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
|
|
runs = RunRepository(session).list_by_campaign(campaign_id)
|
|
data = campaign.model_dump()
|
|
data["progress"] = campaign_progress(campaign, runs)
|
|
return data
|