"""The single cross-run aggregation seam (ADR-0004). Every reader that rolls Runs up into pass_rate / availability / latency — dashboard, scenario ranking, trend, campaign report — calls ``aggregate_runs``. No caller may read ``summary["pass_rate"]`` and re-aggregate on its own. Rules (ADR-0004, extending ADR-0002's service perspective across runs): - a genuinely faulted run counts 0.0 in both pass_rate and availability; - a user-cancelled run (``summary.error.code == "cancelled_by_user"``) is excluded from both denominators — cancellation is a user action, not a quality or availability signal of the target; - ``run_count`` still reports everything that happened, cancelled included. """ from collections import defaultdict from datetime import datetime, timezone from typing import Any from agenteval.models import EvalRun, RunStatus def aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]: scored = [r for r in runs if not (r.summary is not None and r.summary.is_cancelled)] n = len(scored) if n == 0: return {"run_count": len(runs), "pass_rate": None, "availability": None, "avg_latency_ms": None} completed = [r for r in scored if r.status == RunStatus.COMPLETED] pass_rates = [_completed_pass_rate(r) if r.status == RunStatus.COMPLETED else 0.0 for r in scored] latencies = [ r.summary.avg_latency_ms for r in completed if r.summary is not None and r.summary.avg_latency_ms is not None ] return { "run_count": len(runs), "pass_rate": round(sum(pass_rates) / n, 4), "availability": round(len(completed) / n, 4), "avg_latency_ms": round(sum(latencies) / len(latencies), 1) if latencies else None, } def settled_runs(runs: list[EvalRun]) -> list[EvalRun]: """Runs with an outcome — in-flight runs are not results yet. Aggregation itself (fault=0.0, cancelled excluded) is ADR-0004's concern and lives in ``aggregate_runs``; callers only choose *which* runs count. """ return [r for r in runs if r.status in (RunStatus.COMPLETED, RunStatus.FAILED)] def _sortable_ts(dt: datetime | None) -> float: """Sortable timestamp tolerant of naive/aware mixes in legacy rows.""" if dt is None: return 0.0 if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) return dt.timestamp() def compute_dashboard( runs: list[EvalRun], scenario_names: dict[str, str], target_names: dict[str, str], ) -> dict[str, Any]: """Dashboard aggregation: counts, pass rate, per-scenario stats, recent runs. Pure function — no I/O. Caller fetches runs and name maps, passes them in. """ settled = settled_runs(runs) overall_pass_rate = aggregate_runs(settled)["pass_rate"] today = datetime.now(timezone.utc).date() today_runs = 0 running_count = 0 trigger_breakdown: dict[str, int] = defaultdict(int) for r in runs: if r.started_at: started = r.started_at if started.tzinfo is None: started = started.replace(tzinfo=timezone.utc) if started.date() == today: today_runs += 1 if r.status in (RunStatus.RUNNING, RunStatus.PENDING): running_count += 1 trigger_breakdown[r.triggered_by.value] += 1 by_scenario: dict[str, list] = defaultdict(list) for r in settled: by_scenario[r.scenario_id].append(r) scenario_stats = [] for sid, sruns in by_scenario.items(): agg = aggregate_runs(sruns) last_run = max(sruns, key=lambda r: _sortable_ts(r.started_at)) scenario_stats.append({ "scenario_id": sid, "scenario_name": scenario_names.get(sid, sid[:8]), "run_count": agg["run_count"], "avg_pass_rate": agg["pass_rate"], "last_run_at": last_run.started_at.isoformat() if last_run.started_at else None, }) scenario_stats.sort(key=lambda s: s["run_count"], reverse=True) recent_runs = sorted(runs, key=lambda r: _sortable_ts(r.started_at), reverse=True)[:10] return { "runs_count": len(runs), "today_runs": today_runs, "running_count": running_count, "overall_pass_rate": overall_pass_rate, "trigger_breakdown": dict(trigger_breakdown), "scenario_stats": scenario_stats, "recent_runs": [ { **r.model_dump(), "scenario_name": scenario_names.get(r.scenario_id), "target_name": target_names.get(r.target_id), } for r in recent_runs ], } def _completed_pass_rate(run: EvalRun) -> float: if run.summary is None or run.summary.pass_rate is None: return 0.0 return run.summary.pass_rate