782916a283
Give EvalRun.summary a typed RunSummary value (unified RunError, lenient legacy parsing) so readers stop reaching into a schemaless dict, and route every cross-run rollup — dashboard, scenario ranking, trend, campaign report — through one aggregate_runs seam. Fixes the divergence where stats averaged pass_rate over completed-only runs while the campaign report counted faults as 0.0. Cross-run rule (ADR-0004): genuine faults count 0.0, user-cancelled runs are excluded from both denominators.
125 lines
4.3 KiB
Python
125 lines
4.3 KiB
Python
"""API routes for statistics and dashboard data."""
|
|
|
|
from collections import defaultdict
|
|
from datetime import datetime, timezone
|
|
|
|
from fastapi import APIRouter, Depends
|
|
from sqlmodel import Session
|
|
|
|
from agenteval.evaluation.metrics import aggregate_runs
|
|
from agenteval.models import EvalRun, RunStatus
|
|
from agenteval.storage.model_config_repository import ModelConfigRepository
|
|
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
|
from agenteval.web.deps import get_db
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
def _ts(dt: datetime | None) -> float:
|
|
"""Sortable timestamp tolerant of naive/aware mixes in legacy rows."""
|
|
if dt is None:
|
|
return 0.0
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
return dt.timestamp()
|
|
|
|
|
|
def _settled(runs: list[EvalRun]) -> list[EvalRun]:
|
|
"""Runs with an outcome — in-flight runs are not results yet.
|
|
|
|
Aggregation itself (fault=0.0, cancelled excluded) is ADR-0004's concern
|
|
and lives in ``aggregate_runs``; callers only choose *which* runs count.
|
|
"""
|
|
return [r for r in runs if r.status in (RunStatus.COMPLETED, RunStatus.FAILED)]
|
|
|
|
|
|
@router.get("/dashboard")
|
|
def dashboard(session: Session = Depends(get_db)) -> dict:
|
|
targets = TargetRepository(session).list_all()
|
|
scenarios = ScenarioRepository(session).list_all()
|
|
runs = RunRepository(session).list_all()
|
|
model_configs = ModelConfigRepository(session).list_all()
|
|
|
|
scenario_names = {s.id: s.name for s in scenarios}
|
|
target_names = {t.id: t.name for t in targets}
|
|
|
|
settled_runs = _settled(runs)
|
|
overall_pass_rate = aggregate_runs(settled_runs)["pass_rate"]
|
|
|
|
today = datetime.now(timezone.utc).date()
|
|
today_runs = 0
|
|
running_count = 0
|
|
trigger_breakdown: dict[str, int] = defaultdict(int)
|
|
for r in runs:
|
|
if r.started_at:
|
|
started = r.started_at
|
|
if started.tzinfo is None:
|
|
started = started.replace(tzinfo=timezone.utc)
|
|
if started.date() == today:
|
|
today_runs += 1
|
|
if r.status in ("running", "pending"):
|
|
running_count += 1
|
|
trigger_breakdown[r.triggered_by.value] += 1
|
|
|
|
# Per-scenario aggregation over settled runs (ADR-0004 via aggregate_runs).
|
|
by_scenario: dict[str, list] = defaultdict(list)
|
|
for r in settled_runs:
|
|
by_scenario[r.scenario_id].append(r)
|
|
scenario_stats = []
|
|
for sid, sruns in by_scenario.items():
|
|
agg = aggregate_runs(sruns)
|
|
last_run = max(sruns, key=lambda r: _ts(r.started_at))
|
|
scenario_stats.append({
|
|
"scenario_id": sid,
|
|
"scenario_name": scenario_names.get(sid, sid[:8]),
|
|
"run_count": agg["run_count"],
|
|
"avg_pass_rate": agg["pass_rate"],
|
|
"last_run_at": last_run.started_at.isoformat() if last_run.started_at else None,
|
|
})
|
|
scenario_stats.sort(key=lambda s: s["run_count"], reverse=True)
|
|
|
|
recent_runs = sorted(runs, key=lambda r: _ts(r.started_at), reverse=True)[:10]
|
|
|
|
return {
|
|
"targets_count": len(targets),
|
|
"scenarios_count": len(scenarios),
|
|
"runs_count": len(runs),
|
|
"model_configs_count": len(model_configs),
|
|
"today_runs": today_runs,
|
|
"running_count": running_count,
|
|
"overall_pass_rate": overall_pass_rate,
|
|
"trigger_breakdown": dict(trigger_breakdown),
|
|
"scenario_stats": scenario_stats,
|
|
"recent_runs": [
|
|
{
|
|
**r.model_dump(),
|
|
"scenario_name": scenario_names.get(r.scenario_id),
|
|
"target_name": target_names.get(r.target_id),
|
|
}
|
|
for r in recent_runs
|
|
],
|
|
}
|
|
|
|
|
|
@router.get("/trend")
|
|
def trend(days: int = 30, session: Session = Depends(get_db)) -> list[dict]:
|
|
runs = RunRepository(session).list_all()
|
|
|
|
daily: dict[str, list[EvalRun]] = defaultdict(list)
|
|
for run in _settled(runs):
|
|
if run.started_at:
|
|
daily[run.started_at.strftime("%Y-%m-%d")].append(run)
|
|
|
|
sorted_dates = sorted(daily.keys())[-days:]
|
|
points = []
|
|
for d in sorted_dates:
|
|
agg = aggregate_runs(daily[d])
|
|
if agg["pass_rate"] is None: # e.g. only cancelled runs that day
|
|
continue
|
|
points.append({
|
|
"date": d,
|
|
"pass_rate": round(agg["pass_rate"] * 100, 1),
|
|
"run_count": agg["run_count"],
|
|
})
|
|
return points
|