c24998c762
仪表盘聚合逻辑从 stats.py router 下沉到 metrics.py 的 compute_dashboard 纯函数。_settled 重命名为 settled_runs 并公开，_ts 重命名为 _sortable_ts。 router 从 40 行聚合逻辑缩到 5 行，只负责数据获取和序列化。 - 新增 compute_dashboard(runs, scenario_names, target_names) -> dict - 新增 settled_runs(runs) 公开接口（原 _settled） - trend 端点同步迁移到 settled_runs - 5 个新测试覆盖 dashboard 聚合逻辑
128 lines
4.7 KiB
Python
128 lines
4.7 KiB
Python
"""The single cross-run aggregation seam (ADR-0004).
|
|
|
|
Every reader that rolls Runs up into pass_rate / availability / latency —
|
|
dashboard, scenario ranking, trend, campaign report — calls ``aggregate_runs``.
|
|
No caller may read ``summary["pass_rate"]`` and re-aggregate on its own.
|
|
|
|
Rules (ADR-0004, extending ADR-0002's service perspective across runs):
|
|
- a genuinely faulted run counts 0.0 in both pass_rate and availability;
|
|
- a user-cancelled run (``summary.error.code == "cancelled_by_user"``) is
|
|
excluded from both denominators — cancellation is a user action, not a
|
|
quality or availability signal of the target;
|
|
- ``run_count`` still reports everything that happened, cancelled included.
|
|
"""
|
|
|
|
from collections import defaultdict
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
|
|
from agenteval.models import EvalRun, RunStatus
|
|
|
|
|
|
def aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]:
|
|
scored = [r for r in runs if not (r.summary is not None and r.summary.is_cancelled)]
|
|
n = len(scored)
|
|
if n == 0:
|
|
return {"run_count": len(runs), "pass_rate": None, "availability": None, "avg_latency_ms": None}
|
|
|
|
completed = [r for r in scored if r.status == RunStatus.COMPLETED]
|
|
pass_rates = [_completed_pass_rate(r) if r.status == RunStatus.COMPLETED else 0.0 for r in scored]
|
|
latencies = [
|
|
r.summary.avg_latency_ms
|
|
for r in completed
|
|
if r.summary is not None and r.summary.avg_latency_ms is not None
|
|
]
|
|
return {
|
|
"run_count": len(runs),
|
|
"pass_rate": round(sum(pass_rates) / n, 4),
|
|
"availability": round(len(completed) / n, 4),
|
|
"avg_latency_ms": round(sum(latencies) / len(latencies), 1) if latencies else None,
|
|
}
|
|
|
|
|
|
def settled_runs(runs: list[EvalRun]) -> list[EvalRun]:
|
|
"""Runs with an outcome — in-flight runs are not results yet.
|
|
|
|
Aggregation itself (fault=0.0, cancelled excluded) is ADR-0004's concern
|
|
and lives in ``aggregate_runs``; callers only choose *which* runs count.
|
|
"""
|
|
return [r for r in runs if r.status in (RunStatus.COMPLETED, RunStatus.FAILED)]
|
|
|
|
|
|
def _sortable_ts(dt: datetime | None) -> float:
|
|
"""Sortable timestamp tolerant of naive/aware mixes in legacy rows."""
|
|
if dt is None:
|
|
return 0.0
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
return dt.timestamp()
|
|
|
|
|
|
def compute_dashboard(
|
|
runs: list[EvalRun],
|
|
scenario_names: dict[str, str],
|
|
target_names: dict[str, str],
|
|
) -> dict[str, Any]:
|
|
"""Dashboard aggregation: counts, pass rate, per-scenario stats, recent runs.
|
|
|
|
Pure function — no I/O. Caller fetches runs and name maps, passes them in.
|
|
"""
|
|
settled = settled_runs(runs)
|
|
overall_pass_rate = aggregate_runs(settled)["pass_rate"]
|
|
|
|
today = datetime.now(timezone.utc).date()
|
|
today_runs = 0
|
|
running_count = 0
|
|
trigger_breakdown: dict[str, int] = defaultdict(int)
|
|
for r in runs:
|
|
if r.started_at:
|
|
started = r.started_at
|
|
if started.tzinfo is None:
|
|
started = started.replace(tzinfo=timezone.utc)
|
|
if started.date() == today:
|
|
today_runs += 1
|
|
if r.status in (RunStatus.RUNNING, RunStatus.PENDING):
|
|
running_count += 1
|
|
trigger_breakdown[r.triggered_by.value] += 1
|
|
|
|
by_scenario: dict[str, list] = defaultdict(list)
|
|
for r in settled:
|
|
by_scenario[r.scenario_id].append(r)
|
|
scenario_stats = []
|
|
for sid, sruns in by_scenario.items():
|
|
agg = aggregate_runs(sruns)
|
|
last_run = max(sruns, key=lambda r: _sortable_ts(r.started_at))
|
|
scenario_stats.append({
|
|
"scenario_id": sid,
|
|
"scenario_name": scenario_names.get(sid, sid[:8]),
|
|
"run_count": agg["run_count"],
|
|
"avg_pass_rate": agg["pass_rate"],
|
|
"last_run_at": last_run.started_at.isoformat() if last_run.started_at else None,
|
|
})
|
|
scenario_stats.sort(key=lambda s: s["run_count"], reverse=True)
|
|
|
|
recent_runs = sorted(runs, key=lambda r: _sortable_ts(r.started_at), reverse=True)[:10]
|
|
|
|
return {
|
|
"runs_count": len(runs),
|
|
"today_runs": today_runs,
|
|
"running_count": running_count,
|
|
"overall_pass_rate": overall_pass_rate,
|
|
"trigger_breakdown": dict(trigger_breakdown),
|
|
"scenario_stats": scenario_stats,
|
|
"recent_runs": [
|
|
{
|
|
**r.model_dump(),
|
|
"scenario_name": scenario_names.get(r.scenario_id),
|
|
"target_name": target_names.get(r.target_id),
|
|
}
|
|
for r in recent_runs
|
|
],
|
|
}
|
|
|
|
|
|
def _completed_pass_rate(run: EvalRun) -> float:
|
|
if run.summary is None or run.summary.pass_rate is None:
|
|
return 0.0
|
|
return run.summary.pass_rate
|