f285738f6d
report.py mixed DB-reading generation with string formatting: the four render_*_report(run_id, session) functions each re-fetched via generate_report, so the HTML/Markdown/JSON formatting was welded to storage and could not be unit-tested from a plain dict. Extract the formatting into a new pure report_render module whose renderers take the already-built report dict (no session, no storage import). Migrate every caller to generate-then- render, delete the old coupled renderers with no back-compat shim, and drop the _aggregate_runs middle-man alias in favour of metrics.aggregate_runs.
132 lines
5.0 KiB
Python
132 lines
5.0 KiB
Python
"""API routes for evaluation campaigns (评估活动).
|
|
|
|
Creating a campaign starts its durable scheduler loop (``campaign_runner``),
|
|
which spawns child Runs across the (optionally compressed) service-cycle window
|
|
until it finishes. Progress is authoritative in the DB, so detail queries report
|
|
the live window position and spawned/completed Run counts, and a campaign can be
|
|
cancelled mid-flight.
|
|
"""
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Response
|
|
from pydantic import BaseModel, Field
|
|
from sqlmodel import Session
|
|
|
|
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
|
from agenteval.evaluation.report import generate_campaign_report, summarize_campaign_progress
|
|
from agenteval.evaluation.report_render import render_campaign_markdown
|
|
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
|
|
from agenteval.storage.db import utc_now
|
|
from agenteval.storage.repository import (
|
|
CampaignRepository,
|
|
RunRepository,
|
|
ScenarioRepository,
|
|
TargetRepository,
|
|
)
|
|
from agenteval.web.deps import get_db
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
class CreateCampaignRequest(BaseModel):
|
|
name: str
|
|
target_id: str
|
|
window_seconds: int = Field(gt=0)
|
|
time_scale: float = Field(default=1.0, gt=0)
|
|
plan: list[CampaignPlanEntry] = Field(min_length=1)
|
|
|
|
|
|
@router.get("")
|
|
async def list_campaigns(session: Session = Depends(get_db)) -> list[dict]:
|
|
repo = CampaignRepository(session)
|
|
run_repo = RunRepository(session)
|
|
result = []
|
|
for campaign in repo.list_all():
|
|
data = campaign.model_dump()
|
|
runs = run_repo.list_by_campaign(campaign.id)
|
|
data["progress"] = summarize_campaign_progress(campaign, runs)
|
|
result.append(data)
|
|
return result
|
|
|
|
|
|
@router.post("")
|
|
async def create_campaign(
|
|
request: CreateCampaignRequest,
|
|
session: Session = Depends(get_db),
|
|
) -> dict:
|
|
if not TargetRepository(session).get(request.target_id):
|
|
raise HTTPException(status_code=404, detail="target not found")
|
|
|
|
scenario_repo = ScenarioRepository(session)
|
|
for entry in request.plan:
|
|
if not scenario_repo.get(entry.scenario_id):
|
|
raise HTTPException(
|
|
status_code=404,
|
|
detail=f"scenario not found: {entry.scenario_id}",
|
|
)
|
|
|
|
campaign = Campaign(
|
|
name=request.name,
|
|
target_id=request.target_id,
|
|
window_seconds=request.window_seconds,
|
|
time_scale=request.time_scale,
|
|
plan=request.plan,
|
|
)
|
|
repo = CampaignRepository(session)
|
|
campaign = repo.create(campaign)
|
|
# Kick off the durable loop; it moves the campaign into RUNNING.
|
|
start_campaign(campaign.id, session)
|
|
return (repo.get(campaign.id) or campaign).model_dump()
|
|
|
|
|
|
@router.post("/{campaign_id}/cancel")
|
|
async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
|
repo = CampaignRepository(session)
|
|
campaign = repo.get(campaign_id)
|
|
if not campaign:
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
if campaign.status not in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
|
raise HTTPException(status_code=400, detail="campaign is not in a cancellable state")
|
|
|
|
campaign.status = CampaignStatus.CANCELLED
|
|
campaign.completed_at = utc_now()
|
|
repo.update(campaign)
|
|
request_cancel(campaign_id)
|
|
return campaign.model_dump()
|
|
|
|
|
|
@router.get("/{campaign_id}/report")
|
|
async def get_campaign_report(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
|
campaign = CampaignRepository(session).get(campaign_id)
|
|
if not campaign:
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
runs = RunRepository(session).list_by_campaign(campaign_id)
|
|
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
|
return generate_campaign_report(campaign, runs, scenario_names=scenario_names)
|
|
|
|
|
|
@router.get("/{campaign_id}/report/markdown")
|
|
async def get_campaign_report_markdown(campaign_id: str, session: Session = Depends(get_db)) -> Response:
|
|
campaign = CampaignRepository(session).get(campaign_id)
|
|
if not campaign:
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
runs = RunRepository(session).list_by_campaign(campaign_id)
|
|
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
|
md = render_campaign_markdown(generate_campaign_report(campaign, runs, scenario_names=scenario_names))
|
|
return Response(
|
|
content=md,
|
|
media_type="text/markdown; charset=utf-8",
|
|
headers={"Content-Disposition": f'attachment; filename="campaign-report-{campaign_id}.md"'},
|
|
)
|
|
|
|
|
|
@router.get("/{campaign_id}")
|
|
async def get_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
|
campaign = CampaignRepository(session).get(campaign_id)
|
|
if not campaign:
|
|
raise HTTPException(status_code=404, detail="campaign not found")
|
|
|
|
runs = RunRepository(session).list_by_campaign(campaign_id)
|
|
data = campaign.model_dump()
|
|
data["progress"] = campaign_progress(campaign, runs)
|
|
return data
|