c24998c762
仪表盘聚合逻辑从 stats.py router 下沉到 metrics.py 的 compute_dashboard 纯函数。_settled 重命名为 settled_runs 并公开，_ts 重命名为 _sortable_ts。 router 从 40 行聚合逻辑缩到 5 行，只负责数据获取和序列化。 - 新增 compute_dashboard(runs, scenario_names, target_names) -> dict - 新增 settled_runs(runs) 公开接口（原 _settled） - trend 端点同步迁移到 settled_runs - 5 个新测试覆盖 dashboard 聚合逻辑
105 lines
3.8 KiB
Python
105 lines
3.8 KiB
Python
"""Tests for compute_dashboard — the dashboard aggregation logic."""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from agenteval.evaluation.metrics import compute_dashboard, settled_runs
|
|
from agenteval.models import EvalRun, RunStatus, RunTrigger
|
|
|
|
|
|
def _make_run(
|
|
run_id: str,
|
|
status: RunStatus = RunStatus.COMPLETED,
|
|
pass_rate: float = 1.0,
|
|
started_at: datetime | None = None,
|
|
scenario_id: str = "s1",
|
|
triggered_by: RunTrigger = RunTrigger.MANUAL,
|
|
) -> EvalRun:
|
|
return EvalRun(
|
|
id=run_id,
|
|
target_id="t1",
|
|
scenario_id=scenario_id,
|
|
status=status,
|
|
triggered_by=triggered_by,
|
|
started_at=started_at or datetime.now(timezone.utc),
|
|
completed_at=datetime.now(timezone.utc),
|
|
summary={"pass_rate": pass_rate, "total_cases": 1, "passed_cases": int(pass_rate)},
|
|
)
|
|
|
|
|
|
def test_settled_filters_to_completed_and_failed():
|
|
"""settled_runs excludes running/pending/cancelled."""
|
|
runs = [
|
|
_make_run("r1", RunStatus.COMPLETED),
|
|
_make_run("r2", RunStatus.FAILED),
|
|
_make_run("r3", RunStatus.RUNNING),
|
|
_make_run("r4", RunStatus.PENDING),
|
|
]
|
|
settled = settled_runs(runs)
|
|
assert len(settled) == 2
|
|
assert {r.id for r in settled} == {"r1", "r2"}
|
|
|
|
|
|
def test_dashboard_counts_and_pass_rate():
|
|
"""Dashboard returns correct counts and overall pass rate."""
|
|
now = datetime.now(timezone.utc)
|
|
runs = [
|
|
_make_run("r1", pass_rate=1.0, started_at=now),
|
|
_make_run("r2", pass_rate=0.5, started_at=now),
|
|
_make_run("r3", RunStatus.RUNNING, started_at=now),
|
|
]
|
|
result = compute_dashboard(runs, scenario_names={"s1": "Scenario 1"}, target_names={"t1": "Target 1"})
|
|
assert result["runs_count"] == 3
|
|
assert result["running_count"] == 1
|
|
assert result["today_runs"] == 3 # all runs started today (including running)
|
|
assert result["overall_pass_rate"] == 0.75 # (1.0 + 0.5) / 2
|
|
|
|
|
|
def test_dashboard_trigger_breakdown():
|
|
"""Dashboard groups runs by trigger source."""
|
|
now = datetime.now(timezone.utc)
|
|
runs = [
|
|
_make_run("r1", triggered_by=RunTrigger.MANUAL, started_at=now),
|
|
_make_run("r2", triggered_by=RunTrigger.MANUAL, started_at=now),
|
|
_make_run("r3", triggered_by=RunTrigger.CAMPAIGN, started_at=now),
|
|
]
|
|
result = compute_dashboard(runs, scenario_names={}, target_names={})
|
|
assert result["trigger_breakdown"] == {"manual": 2, "campaign": 1}
|
|
|
|
|
|
def test_dashboard_scenario_stats():
|
|
"""Dashboard aggregates per-scenario stats."""
|
|
now = datetime.now(timezone.utc)
|
|
runs = [
|
|
_make_run("r1", scenario_id="s1", pass_rate=1.0, started_at=now),
|
|
_make_run("r2", scenario_id="s1", pass_rate=0.5, started_at=now - timedelta(hours=1)),
|
|
_make_run("r3", scenario_id="s2", pass_rate=0.8, started_at=now),
|
|
]
|
|
result = compute_dashboard(
|
|
runs,
|
|
scenario_names={"s1": "Scenario 1", "s2": "Scenario 2"},
|
|
target_names={},
|
|
)
|
|
stats = result["scenario_stats"]
|
|
assert len(stats) == 2
|
|
# s1 has 2 runs, s2 has 1 run — sorted by run_count desc
|
|
assert stats[0]["scenario_id"] == "s1"
|
|
assert stats[0]["run_count"] == 2
|
|
assert stats[0]["avg_pass_rate"] == 0.75
|
|
assert stats[1]["scenario_id"] == "s2"
|
|
assert stats[1]["run_count"] == 1
|
|
|
|
|
|
def test_dashboard_recent_runs_sorted_and_limited():
|
|
"""Dashboard returns 10 most recent runs, sorted by started_at desc."""
|
|
now = datetime.now(timezone.utc)
|
|
runs = [
|
|
_make_run(f"r{i}", started_at=now - timedelta(hours=i))
|
|
for i in range(15)
|
|
]
|
|
result = compute_dashboard(runs, scenario_names={}, target_names={})
|
|
recent = result["recent_runs"]
|
|
assert len(recent) == 10
|
|
# Most recent first
|
|
assert recent[0]["id"] == "r0"
|
|
assert recent[9]["id"] == "r9"
|