refactor(metrics): extract dashboard aggregation to compute_dashboard
仪表盘聚合逻辑从 stats.py router 下沉到 metrics.py 的 compute_dashboard 纯函数。_settled 重命名为 settled_runs 并公开,_ts 重命名为 _sortable_ts。 router 从 40 行聚合逻辑缩到 5 行,只负责数据获取和序列化。 - 新增 compute_dashboard(runs, scenario_names, target_names) -> dict - 新增 settled_runs(runs) 公开接口(原 _settled) - trend 端点同步迁移到 settled_runs - 5 个新测试覆盖 dashboard 聚合逻辑
This commit is contained in:
parent
42be31dd1f
commit
c24998c762
@ -12,6 +12,8 @@ Rules (ADR-0004, extending ADR-0002's service perspective across runs):
|
||||
- ``run_count`` still reports everything that happened, cancelled included.
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from agenteval.models import EvalRun, RunStatus
|
||||
@ -38,6 +40,87 @@ def aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def settled_runs(runs: list[EvalRun]) -> list[EvalRun]:
|
||||
"""Runs with an outcome — in-flight runs are not results yet.
|
||||
|
||||
Aggregation itself (fault=0.0, cancelled excluded) is ADR-0004's concern
|
||||
and lives in ``aggregate_runs``; callers only choose *which* runs count.
|
||||
"""
|
||||
return [r for r in runs if r.status in (RunStatus.COMPLETED, RunStatus.FAILED)]
|
||||
|
||||
|
||||
def _sortable_ts(dt: datetime | None) -> float:
|
||||
"""Sortable timestamp tolerant of naive/aware mixes in legacy rows."""
|
||||
if dt is None:
|
||||
return 0.0
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.timestamp()
|
||||
|
||||
|
||||
def compute_dashboard(
|
||||
runs: list[EvalRun],
|
||||
scenario_names: dict[str, str],
|
||||
target_names: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
"""Dashboard aggregation: counts, pass rate, per-scenario stats, recent runs.
|
||||
|
||||
Pure function — no I/O. Caller fetches runs and name maps, passes them in.
|
||||
"""
|
||||
settled = settled_runs(runs)
|
||||
overall_pass_rate = aggregate_runs(settled)["pass_rate"]
|
||||
|
||||
today = datetime.now(timezone.utc).date()
|
||||
today_runs = 0
|
||||
running_count = 0
|
||||
trigger_breakdown: dict[str, int] = defaultdict(int)
|
||||
for r in runs:
|
||||
if r.started_at:
|
||||
started = r.started_at
|
||||
if started.tzinfo is None:
|
||||
started = started.replace(tzinfo=timezone.utc)
|
||||
if started.date() == today:
|
||||
today_runs += 1
|
||||
if r.status in (RunStatus.RUNNING, RunStatus.PENDING):
|
||||
running_count += 1
|
||||
trigger_breakdown[r.triggered_by.value] += 1
|
||||
|
||||
by_scenario: dict[str, list] = defaultdict(list)
|
||||
for r in settled:
|
||||
by_scenario[r.scenario_id].append(r)
|
||||
scenario_stats = []
|
||||
for sid, sruns in by_scenario.items():
|
||||
agg = aggregate_runs(sruns)
|
||||
last_run = max(sruns, key=lambda r: _sortable_ts(r.started_at))
|
||||
scenario_stats.append({
|
||||
"scenario_id": sid,
|
||||
"scenario_name": scenario_names.get(sid, sid[:8]),
|
||||
"run_count": agg["run_count"],
|
||||
"avg_pass_rate": agg["pass_rate"],
|
||||
"last_run_at": last_run.started_at.isoformat() if last_run.started_at else None,
|
||||
})
|
||||
scenario_stats.sort(key=lambda s: s["run_count"], reverse=True)
|
||||
|
||||
recent_runs = sorted(runs, key=lambda r: _sortable_ts(r.started_at), reverse=True)[:10]
|
||||
|
||||
return {
|
||||
"runs_count": len(runs),
|
||||
"today_runs": today_runs,
|
||||
"running_count": running_count,
|
||||
"overall_pass_rate": overall_pass_rate,
|
||||
"trigger_breakdown": dict(trigger_breakdown),
|
||||
"scenario_stats": scenario_stats,
|
||||
"recent_runs": [
|
||||
{
|
||||
**r.model_dump(),
|
||||
"scenario_name": scenario_names.get(r.scenario_id),
|
||||
"target_name": target_names.get(r.target_id),
|
||||
}
|
||||
for r in recent_runs
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _completed_pass_rate(run: EvalRun) -> float:
|
||||
if run.summary is None or run.summary.pass_rate is None:
|
||||
return 0.0
|
||||
|
||||
@ -1,13 +1,9 @@
|
||||
"""API routes for statistics and dashboard data."""
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlmodel import Session
|
||||
|
||||
from agenteval.evaluation.metrics import aggregate_runs
|
||||
from agenteval.models import EvalRun, RunStatus
|
||||
from agenteval.evaluation.metrics import compute_dashboard, settled_runs
|
||||
from agenteval.storage.model_config_repository import ModelConfigRepository
|
||||
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
||||
from agenteval.web.deps import get_db
|
||||
@ -15,24 +11,6 @@ from agenteval.web.deps import get_db
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _ts(dt: datetime | None) -> float:
|
||||
"""Sortable timestamp tolerant of naive/aware mixes in legacy rows."""
|
||||
if dt is None:
|
||||
return 0.0
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.timestamp()
|
||||
|
||||
|
||||
def _settled(runs: list[EvalRun]) -> list[EvalRun]:
|
||||
"""Runs with an outcome — in-flight runs are not results yet.
|
||||
|
||||
Aggregation itself (fault=0.0, cancelled excluded) is ADR-0004's concern
|
||||
and lives in ``aggregate_runs``; callers only choose *which* runs count.
|
||||
"""
|
||||
return [r for r in runs if r.status in (RunStatus.COMPLETED, RunStatus.FAILED)]
|
||||
|
||||
|
||||
@router.get("/dashboard")
|
||||
def dashboard(session: Session = Depends(get_db)) -> dict:
|
||||
targets = TargetRepository(session).list_all()
|
||||
@ -42,70 +20,25 @@ def dashboard(session: Session = Depends(get_db)) -> dict:
|
||||
|
||||
target_names = {t.id: t.name for t in targets}
|
||||
|
||||
settled_runs = _settled(runs)
|
||||
overall_pass_rate = aggregate_runs(settled_runs)["pass_rate"]
|
||||
|
||||
today = datetime.now(timezone.utc).date()
|
||||
today_runs = 0
|
||||
running_count = 0
|
||||
trigger_breakdown: dict[str, int] = defaultdict(int)
|
||||
for r in runs:
|
||||
if r.started_at:
|
||||
started = r.started_at
|
||||
if started.tzinfo is None:
|
||||
started = started.replace(tzinfo=timezone.utc)
|
||||
if started.date() == today:
|
||||
today_runs += 1
|
||||
if r.status in ("running", "pending"):
|
||||
running_count += 1
|
||||
trigger_breakdown[r.triggered_by.value] += 1
|
||||
|
||||
# Per-scenario aggregation over settled runs (ADR-0004 via aggregate_runs).
|
||||
by_scenario: dict[str, list] = defaultdict(list)
|
||||
for r in settled_runs:
|
||||
by_scenario[r.scenario_id].append(r)
|
||||
scenario_stats = []
|
||||
for sid, sruns in by_scenario.items():
|
||||
agg = aggregate_runs(sruns)
|
||||
last_run = max(sruns, key=lambda r: _ts(r.started_at))
|
||||
scenario_stats.append({
|
||||
"scenario_id": sid,
|
||||
"scenario_name": scenario_names.get(sid, sid[:8]),
|
||||
"run_count": agg["run_count"],
|
||||
"avg_pass_rate": agg["pass_rate"],
|
||||
"last_run_at": last_run.started_at.isoformat() if last_run.started_at else None,
|
||||
})
|
||||
scenario_stats.sort(key=lambda s: s["run_count"], reverse=True)
|
||||
|
||||
recent_runs = sorted(runs, key=lambda r: _ts(r.started_at), reverse=True)[:10]
|
||||
|
||||
dashboard_data = compute_dashboard(runs, scenario_names, target_names)
|
||||
return {
|
||||
"targets_count": len(targets),
|
||||
"scenarios_count": len(scenario_names),
|
||||
"runs_count": len(runs),
|
||||
"model_configs_count": len(model_configs),
|
||||
"today_runs": today_runs,
|
||||
"running_count": running_count,
|
||||
"overall_pass_rate": overall_pass_rate,
|
||||
"trigger_breakdown": dict(trigger_breakdown),
|
||||
"scenario_stats": scenario_stats,
|
||||
"recent_runs": [
|
||||
{
|
||||
**r.model_dump(),
|
||||
"scenario_name": scenario_names.get(r.scenario_id),
|
||||
"target_name": target_names.get(r.target_id),
|
||||
}
|
||||
for r in recent_runs
|
||||
],
|
||||
**dashboard_data,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/trend")
|
||||
def trend(days: int = 30, session: Session = Depends(get_db)) -> list[dict]:
|
||||
from collections import defaultdict
|
||||
|
||||
from agenteval.evaluation.metrics import aggregate_runs
|
||||
|
||||
runs = RunRepository(session).list_all()
|
||||
|
||||
daily: dict[str, list[EvalRun]] = defaultdict(list)
|
||||
for run in _settled(runs):
|
||||
daily: dict[str, list] = defaultdict(list)
|
||||
for run in settled_runs(runs):
|
||||
if run.started_at:
|
||||
daily[run.started_at.strftime("%Y-%m-%d")].append(run)
|
||||
|
||||
|
||||
104
tests/unit/test_dashboard_metrics.py
Normal file
104
tests/unit/test_dashboard_metrics.py
Normal file
@ -0,0 +1,104 @@
|
||||
"""Tests for compute_dashboard — the dashboard aggregation logic."""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from agenteval.evaluation.metrics import compute_dashboard, settled_runs
|
||||
from agenteval.models import EvalRun, RunStatus, RunTrigger
|
||||
|
||||
|
||||
def _make_run(
|
||||
run_id: str,
|
||||
status: RunStatus = RunStatus.COMPLETED,
|
||||
pass_rate: float = 1.0,
|
||||
started_at: datetime | None = None,
|
||||
scenario_id: str = "s1",
|
||||
triggered_by: RunTrigger = RunTrigger.MANUAL,
|
||||
) -> EvalRun:
|
||||
return EvalRun(
|
||||
id=run_id,
|
||||
target_id="t1",
|
||||
scenario_id=scenario_id,
|
||||
status=status,
|
||||
triggered_by=triggered_by,
|
||||
started_at=started_at or datetime.now(timezone.utc),
|
||||
completed_at=datetime.now(timezone.utc),
|
||||
summary={"pass_rate": pass_rate, "total_cases": 1, "passed_cases": int(pass_rate)},
|
||||
)
|
||||
|
||||
|
||||
def test_settled_filters_to_completed_and_failed():
|
||||
"""settled_runs excludes running/pending/cancelled."""
|
||||
runs = [
|
||||
_make_run("r1", RunStatus.COMPLETED),
|
||||
_make_run("r2", RunStatus.FAILED),
|
||||
_make_run("r3", RunStatus.RUNNING),
|
||||
_make_run("r4", RunStatus.PENDING),
|
||||
]
|
||||
settled = settled_runs(runs)
|
||||
assert len(settled) == 2
|
||||
assert {r.id for r in settled} == {"r1", "r2"}
|
||||
|
||||
|
||||
def test_dashboard_counts_and_pass_rate():
|
||||
"""Dashboard returns correct counts and overall pass rate."""
|
||||
now = datetime.now(timezone.utc)
|
||||
runs = [
|
||||
_make_run("r1", pass_rate=1.0, started_at=now),
|
||||
_make_run("r2", pass_rate=0.5, started_at=now),
|
||||
_make_run("r3", RunStatus.RUNNING, started_at=now),
|
||||
]
|
||||
result = compute_dashboard(runs, scenario_names={"s1": "Scenario 1"}, target_names={"t1": "Target 1"})
|
||||
assert result["runs_count"] == 3
|
||||
assert result["running_count"] == 1
|
||||
assert result["today_runs"] == 3 # all runs started today (including running)
|
||||
assert result["overall_pass_rate"] == 0.75 # (1.0 + 0.5) / 2
|
||||
|
||||
|
||||
def test_dashboard_trigger_breakdown():
|
||||
"""Dashboard groups runs by trigger source."""
|
||||
now = datetime.now(timezone.utc)
|
||||
runs = [
|
||||
_make_run("r1", triggered_by=RunTrigger.MANUAL, started_at=now),
|
||||
_make_run("r2", triggered_by=RunTrigger.MANUAL, started_at=now),
|
||||
_make_run("r3", triggered_by=RunTrigger.CAMPAIGN, started_at=now),
|
||||
]
|
||||
result = compute_dashboard(runs, scenario_names={}, target_names={})
|
||||
assert result["trigger_breakdown"] == {"manual": 2, "campaign": 1}
|
||||
|
||||
|
||||
def test_dashboard_scenario_stats():
|
||||
"""Dashboard aggregates per-scenario stats."""
|
||||
now = datetime.now(timezone.utc)
|
||||
runs = [
|
||||
_make_run("r1", scenario_id="s1", pass_rate=1.0, started_at=now),
|
||||
_make_run("r2", scenario_id="s1", pass_rate=0.5, started_at=now - timedelta(hours=1)),
|
||||
_make_run("r3", scenario_id="s2", pass_rate=0.8, started_at=now),
|
||||
]
|
||||
result = compute_dashboard(
|
||||
runs,
|
||||
scenario_names={"s1": "Scenario 1", "s2": "Scenario 2"},
|
||||
target_names={},
|
||||
)
|
||||
stats = result["scenario_stats"]
|
||||
assert len(stats) == 2
|
||||
# s1 has 2 runs, s2 has 1 run — sorted by run_count desc
|
||||
assert stats[0]["scenario_id"] == "s1"
|
||||
assert stats[0]["run_count"] == 2
|
||||
assert stats[0]["avg_pass_rate"] == 0.75
|
||||
assert stats[1]["scenario_id"] == "s2"
|
||||
assert stats[1]["run_count"] == 1
|
||||
|
||||
|
||||
def test_dashboard_recent_runs_sorted_and_limited():
|
||||
"""Dashboard returns 10 most recent runs, sorted by started_at desc."""
|
||||
now = datetime.now(timezone.utc)
|
||||
runs = [
|
||||
_make_run(f"r{i}", started_at=now - timedelta(hours=i))
|
||||
for i in range(15)
|
||||
]
|
||||
result = compute_dashboard(runs, scenario_names={}, target_names={})
|
||||
recent = result["recent_runs"]
|
||||
assert len(recent) == 10
|
||||
# Most recent first
|
||||
assert recent[0]["id"] == "r0"
|
||||
assert recent[9]["id"] == "r9"
|
||||
Loading…
Reference in New Issue
Block a user