refactor(metrics): type Run summary and converge cross-run aggregation

Give EvalRun.summary a typed RunSummary value (unified RunError, lenient
legacy parsing) so readers stop reaching into a schemaless dict, and route
every cross-run rollup — dashboard, scenario ranking, trend, campaign
report — through one aggregate_runs seam. Fixes the divergence where
stats averaged pass_rate over completed-only runs while the campaign
report counted faults as 0.0. Cross-run rule (ADR-0004): genuine faults
count 0.0, user-cancelled runs are excluded from both denominators.

backend/agenteval/evaluation/metrics.py

@@ -0,0 +1,44 @@
+"""The single cross-run aggregation seam (ADR-0004).
+
+Every reader that rolls Runs up into pass_rate / availability / latency —
+dashboard, scenario ranking, trend, campaign report — calls ``aggregate_runs``.
+No caller may read ``summary["pass_rate"]`` and re-aggregate on its own.
+
+Rules (ADR-0004, extending ADR-0002's service perspective across runs):
+- a genuinely faulted run counts 0.0 in both pass_rate and availability;
+- a user-cancelled run (``summary.error.code == "cancelled_by_user"``) is
+  excluded from both denominators — cancellation is a user action, not a
+  quality or availability signal of the target;
+- ``run_count`` still reports everything that happened, cancelled included.
+"""
+
+from typing import Any
+
+from agenteval.models import EvalRun, RunStatus
+
+
+def aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]:
+    scored = [r for r in runs if not (r.summary is not None and r.summary.is_cancelled)]
+    n = len(scored)
+    if n == 0:
+        return {"run_count": len(runs), "pass_rate": None, "availability": None, "avg_latency_ms": None}
+
+    completed = [r for r in scored if r.status == RunStatus.COMPLETED]
+    pass_rates = [_completed_pass_rate(r) if r.status == RunStatus.COMPLETED else 0.0 for r in scored]
+    latencies = [
+        r.summary.avg_latency_ms
+        for r in completed
+        if r.summary is not None and r.summary.avg_latency_ms is not None
+    ]
+    return {
+        "run_count": len(runs),
+        "pass_rate": round(sum(pass_rates) / n, 4),
+        "availability": round(len(completed) / n, 4),
+        "avg_latency_ms": round(sum(latencies) / len(latencies), 1) if latencies else None,
+    }
+
+
+def _completed_pass_rate(run: EvalRun) -> float:
+    if run.summary is None or run.summary.pass_rate is None:
+        return 0.0
+    return run.summary.pass_rate

backend/agenteval/evaluation/report.py

@@ -8,7 +8,8 @@ from typing import Any, Optional
 
 from jinja2 import Template
 
-from agenteval.models import Campaign, EvalRun, RunStatus
+from agenteval.evaluation.metrics import aggregate_runs
+from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary
 from agenteval.storage.db import DATA_DIR, iso_utc
 from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
 from agenteval.utils.llm import extract_reply_text
@@ -139,19 +140,19 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
             }
         )
 
-    summary = run.summary or {}
+    summary = run.summary or RunSummary()
-    errored_case_ids = {e.get("case_id") for e in summary.get("case_errors", [])}
+    errored_case_ids = {e.get("case_id") for e in summary.case_errors}
     # 权威判定：引擎经 judgement.combine_case_outcome 算一次写入 summary；
     # 老 run 没有该字段时退回从持久化结果反推（WEIGHTED/ANY 只能近似）。
-    authoritative: dict[str, Any] = summary.get("case_outcomes", {})
+    authoritative = summary.case_outcomes
 
     cases = []
     for case_id in sorted(case_map.keys()):
         item = case_map[case_id]
         if case_id in authoritative:
             outcome = authoritative[case_id]
-            connectivity = bool(outcome.get("connectivity"))
+            connectivity = outcome.connectivity
-            passed = bool(outcome.get("passed"))
+            passed = outcome.passed
         else:
             # 连通用例：无任何判定结果，且每轮都收到回复、无用例级错误（CONTEXT.md）
             connectivity = (
@@ -177,8 +178,8 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
             }
         )
 
-    total_cases = summary.get("total_cases", 0)
+    total_cases = summary.total_cases
-    passed_cases = summary.get("passed_cases", 0)
+    passed_cases = summary.passed_cases
     connectivity_count = sum(1 for c in cases if c["connectivity"])
     judged_total = total_cases - connectivity_count
     # 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数
@@ -198,10 +199,10 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
         "summary": {
             "total_cases": total_cases,
             "passed_cases": passed_cases,
-            "failed_cases": summary.get("failed_cases", 0),
+            "failed_cases": summary.failed_cases,
-            "total_rules": summary.get("total_rules", 0),
+            "total_rules": summary.total_rules,
-            "passed_rules": summary.get("passed_rules", 0),
+            "passed_rules": summary.passed_rules,
-            "pass_rate": summary.get("pass_rate", 0.0),
+            "pass_rate": summary.pass_rate if summary.pass_rate is not None else 0.0,
             "connectivity_cases": connectivity_count,
             "judged_pass_rate": judged_pass_rate,
         },
@@ -292,35 +293,8 @@ def _to_utc(dt: Optional[datetime]) -> Optional[datetime]:
 
 
 def _aggregate_runs(runs: list[EvalRun]) -> dict[str, Any]:
-    """Aggregate a set of child Runs into pass_rate / availability / latency.
+    """Delegates to the single cross-run aggregation seam (ADR-0004)."""
-
+    return aggregate_runs(runs)
-    pass_rate follows the single-Run case-level rate and *includes execution
-    failures* (ADR-0002): a failed child Run contributes 0.0, so a bad time
-    slice drags the curve down. availability is the completed fraction (an
-    orthogonal execution-success signal). latency averages only completed Runs
-    that recorded one. time_scale never enters these numbers.
-    """
-    n = len(runs)
-    if n == 0:
-        return {"run_count": 0, "pass_rate": None, "availability": None, "avg_latency_ms": None}
-    completed = [r for r in runs if r.status == RunStatus.COMPLETED]
-    pass_rates = [
-        (r.summary or {}).get("pass_rate", 0.0)
-        if (r.status == RunStatus.COMPLETED and isinstance(r.summary, dict))
-        else 0.0
-        for r in runs
-    ]
-    latencies = [
-        r.summary["avg_latency_ms"]
-        for r in completed
-        if isinstance(r.summary, dict) and r.summary.get("avg_latency_ms") is not None
-    ]
-    return {
-        "run_count": n,
-        "pass_rate": round(sum(pass_rates) / n, 4),
-        "availability": round(len(completed) / n, 4),
-        "avg_latency_ms": round(sum(latencies) / len(latencies), 1) if latencies else None,
-    }
 
 
 def summarize_campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict[str, Any]:

backend/agenteval/models.py

@@ -157,9 +157,61 @@ class RunTrigger(str, Enum):
     CAMPAIGN = "campaign"
 
 
+class RunError(BaseModel):
+    """Unified run-level error: user cancellation vs genuine execution fault."""
+
+    code: str = "error"
+    message: str = ""
+
+
+class CaseOutcomeSummary(BaseModel):
+    """Per-case authoritative verdict snapshot stored in the run summary."""
+
+    passed: bool = False
+    connectivity: bool = False
+
+
+class RunSummary(BaseModel):
+    """Typed value of ``EvalRun.summary`` — the single interface for its keys.
+
+    All fields are defaulted and unknown keys are preserved so summary dicts
+    written by older versions keep parsing (and survive read-modify-write).
+    """
+
+    model_config = {"extra": "allow"}
+
+    total_cases: int = 0
+    passed_cases: int = 0
+    failed_cases: int = 0
+    total_rules: int = 0
+    passed_rules: int = 0
+    # 用例级通过率，含执行失败（ADR-0002）；失败/取消的 run 无此值
+    pass_rate: Optional[float] = None
+    avg_latency_ms: Optional[float] = None
+    case_outcomes: dict[str, CaseOutcomeSummary] = Field(default_factory=dict)
+    case_errors: list[dict[str, str]] = Field(default_factory=list)
+    model_configs: dict[str, Any] = Field(default_factory=dict)
+    error: Optional[RunError] = None
+
+    @field_validator("error", mode="before")
+    @classmethod
+    def _coerce_legacy_error(cls, v: Any) -> Any:
+        if isinstance(v, str):
+            return {"code": "error", "message": v}
+        return v
+
+    @property
+    def is_cancelled(self) -> bool:
+        """User-initiated cancellation — excluded from aggregation (ADR-0004)."""
+        return self.error is not None and self.error.code == "cancelled_by_user"
+
+
 class EvalRun(BaseModel):
     """A single evaluation run."""
 
+    # summary 以属性赋值写入（engine/routers），赋值时即校验成 RunSummary
+    model_config = {"validate_assignment": True}
+
     id: Optional[str] = None
     target_id: str
     scenario_id: str
@@ -171,7 +223,7 @@ class EvalRun(BaseModel):
     triggered_by: RunTrigger = RunTrigger.MANUAL
     started_at: Optional[datetime] = None
     completed_at: Optional[datetime] = None
-    summary: Optional[dict[str, Any]] = None
+    summary: Optional[RunSummary] = None
 
 
 class CampaignStatus(str, Enum):

backend/agenteval/storage/repository.py

@@ -91,8 +91,8 @@ def _run_to_db(run: EvalRun) -> EvalRunDB:
         started_at=run.started_at,
         completed_at=run.completed_at,
     )
-    if run.summary:
+    if run.summary is not None:
-        db.set_summary(run.summary)
+        db.set_summary(run.summary.model_dump(mode="json"))
     return db
 
 
@@ -352,8 +352,8 @@ class RunRepository:
         existing.status = run.status.value
         existing.triggered_by = run.triggered_by.value
         existing.completed_at = run.completed_at
-        if run.summary:
+        if run.summary is not None:
-            existing.set_summary(run.summary)
+            existing.set_summary(run.summary.model_dump(mode="json"))
         self.session.add(existing)
         self.session.commit()
         self.session.refresh(existing)

backend/agenteval/web/routers/runs.py

@@ -60,7 +60,7 @@ async def _run_evaluation(run_id: str, target_id: str, scenario_id: str) -> None
             await send_run_webhook(
                 run_id=run_id,
                 status=completed_run.status.value,
-                summary=completed_run.summary or {},
+                summary=completed_run.summary.model_dump(mode="json") if completed_run.summary else {},
             )
     finally:
         session.close()

backend/agenteval/web/routers/stats.py

@@ -6,6 +6,8 @@ from datetime import datetime, timezone
 from fastapi import APIRouter, Depends
 from sqlmodel import Session
 
+from agenteval.evaluation.metrics import aggregate_runs
+from agenteval.models import EvalRun, RunStatus
 from agenteval.storage.model_config_repository import ModelConfigRepository
 from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
 from agenteval.web.deps import get_db
@@ -22,6 +24,15 @@ def _ts(dt: datetime | None) -> float:
     return dt.timestamp()
 
 
+def _settled(runs: list[EvalRun]) -> list[EvalRun]:
+    """Runs with an outcome — in-flight runs are not results yet.
+
+    Aggregation itself (fault=0.0, cancelled excluded) is ADR-0004's concern
+    and lives in ``aggregate_runs``; callers only choose *which* runs count.
+    """
+    return [r for r in runs if r.status in (RunStatus.COMPLETED, RunStatus.FAILED)]
+
+
 @router.get("/dashboard")
 def dashboard(session: Session = Depends(get_db)) -> dict:
     targets = TargetRepository(session).list_all()
@@ -32,9 +43,8 @@ def dashboard(session: Session = Depends(get_db)) -> dict:
     scenario_names = {s.id: s.name for s in scenarios}
     target_names = {t.id: t.name for t in targets}
 
-    completed_runs = [r for r in runs if r.status == "completed" and r.summary]
+    settled_runs = _settled(runs)
-    pass_rates = [r.summary.get("pass_rate", 0) for r in completed_runs if isinstance(r.summary, dict)]
+    overall_pass_rate = aggregate_runs(settled_runs)["pass_rate"]
-    overall_pass_rate = sum(pass_rates) / len(pass_rates) if pass_rates else None
 
     today = datetime.now(timezone.utc).date()
     today_runs = 0
@@ -51,19 +61,19 @@ def dashboard(session: Session = Depends(get_db)) -> dict:
             running_count += 1
         trigger_breakdown[r.triggered_by.value] += 1
 
-    # Per-scenario aggregation over completed runs.
+    # Per-scenario aggregation over settled runs (ADR-0004 via aggregate_runs).
     by_scenario: dict[str, list] = defaultdict(list)
-    for r in completed_runs:
+    for r in settled_runs:
         by_scenario[r.scenario_id].append(r)
     scenario_stats = []
     for sid, sruns in by_scenario.items():
-        rates = [r.summary.get("pass_rate", 0) for r in sruns if isinstance(r.summary, dict)]
+        agg = aggregate_runs(sruns)
         last_run = max(sruns, key=lambda r: _ts(r.started_at))
         scenario_stats.append({
             "scenario_id": sid,
             "scenario_name": scenario_names.get(sid, sid[:8]),
-            "run_count": len(sruns),
+            "run_count": agg["run_count"],
-            "avg_pass_rate": round(sum(rates) / len(rates), 4) if rates else None,
+            "avg_pass_rate": agg["pass_rate"],
             "last_run_at": last_run.started_at.isoformat() if last_run.started_at else None,
         })
     scenario_stats.sort(key=lambda s: s["run_count"], reverse=True)
@@ -94,20 +104,21 @@ def dashboard(session: Session = Depends(get_db)) -> dict:
 @router.get("/trend")
 def trend(days: int = 30, session: Session = Depends(get_db)) -> list[dict]:
     runs = RunRepository(session).list_all()
-    completed = [r for r in runs if r.status == "completed" and r.summary and r.started_at]
 
-    daily: dict[str, list[float]] = defaultdict(list)
+    daily: dict[str, list[EvalRun]] = defaultdict(list)
-    for run in completed:
+    for run in _settled(runs):
-        date_str = run.started_at.strftime("%Y-%m-%d") if run.started_at else ""
+        if run.started_at:
-        if date_str and isinstance(run.summary, dict):
+            daily[run.started_at.strftime("%Y-%m-%d")].append(run)
-            daily[date_str].append(run.summary.get("pass_rate", 0))
 
     sorted_dates = sorted(daily.keys())[-days:]
-    return [
+    points = []
-        {
+    for d in sorted_dates:
+        agg = aggregate_runs(daily[d])
+        if agg["pass_rate"] is None:  # e.g. only cancelled runs that day
+            continue
+        points.append({
             "date": d,
-            "pass_rate": round(sum(daily[d]) / len(daily[d]) * 100, 1),
+            "pass_rate": round(agg["pass_rate"] * 100, 1),
-            "run_count": len(daily[d]),
+            "run_count": agg["run_count"],
-        }
+        })
-        for d in sorted_dates
+    return points
-    ]

docs/adr/0004-cross-run-aggregation-cancelled-excluded.md

@@ -0,0 +1,20 @@
+# 跨 Run 通过率聚合：故障计零、用户取消排除
+
+ADR-0002 定义了单次 Run 内的通过率口径（用例级、含执行失败），但没有覆盖**跨 Run 聚合**（仪表盘总览、场景排行、趋势、活动周期报告）。实际代码里长出了两个矛盾口径：仪表盘对"仅已完成的 Run"求均值（故障 Run 被排除出分母），活动报告把失败 Run 计 0.0（除以全部）。决定统一为一个聚合规则，由单一函数实现，所有跨 Run 读者共用：
+
+- **执行故障的 Run 计 0.0**——延续 ADR-0002 的服务视角：整个 Run 挂了，等价于该 Run 的所有用例都不达标；
+- **用户手动取消的 Run 排除出分母**（含通过率与可用性）——取消是用户操作，不是被评对象的质量或可用性信号，计 0.0 会诬陷对象，计 1.0 会粉饰，排除是唯一诚实的处理；
+- 取消与故障通过 `RunSummary.error.code == "cancelled_by_user"` 区分（取消 Run 的持久化状态仍是 failed）。
+
+## Considered Options
+
+- **仅对已完成 Run 求均值（仪表盘现状）** — 被否：故障期间仪表盘显示"质量正常"，与 ADR-0002 的动机直接冲突；同一个 pass_rate 键在两处得出不同数字，无从解释。
+- **非 completed 一律计 0.0（活动现状，单一规则最简）** — 被否：用户自己停掉的 Run 拉低质量分，指标不诚实；长期会让人不敢取消 Run。
+- **保留两种聚合、显式命名（service_pass_rate / completed_pass_rate）** — 被否：两个函数就是两个口径，仪表盘与活动报告的数字依旧不可互相印证，分歧只是换了名字。
+
+## Consequences
+
+- 仪表盘 / 场景排行 / 趋势的数字会变：故障 Run 并入分母（数字下降），取消 Run 排除（略回升）。这是修正，不是回归。
+- 活动周期报告的口径同步细化：用户取消的子 Run 不再计 0.0（此前极少发生，现有报告几乎不变）。
+- 聚合逻辑必须收敛在一个函数里；任何新读者（未来的对比视图、导出）禁止自行读 `summary["pass_rate"]` 重新聚合。
+- 取消的判别依赖统一的 `error.code`——错误形状（此前字符串/对象混用）必须随之统一。

frontend/web/src/api.ts

@@ -137,6 +137,24 @@ export interface ModelConfigReference {
 
 export type RunTrigger = 'manual' | 'ai_assistant' | 'cli' | 'campaign'
 
+export interface RunError {
+  code: string
+  message: string
+}
+
+export interface RunSummary {
+  total_cases: number
+  passed_cases: number
+  failed_cases: number
+  total_rules: number
+  passed_rules: number
+  pass_rate: number | null
+  avg_latency_ms: number | null
+  case_outcomes: Record<string, { passed: boolean; connectivity: boolean }>
+  case_errors: Array<Record<string, string>>
+  error: RunError | null
+}
+
 export interface Run {
   id: string
   target_id: string
@@ -149,7 +167,7 @@ export interface Run {
   target_name?: string | null
   started_at: string
   completed_at: string | null
-  summary: Record<string, unknown> | null
+  summary: RunSummary | null
 }
 
 export interface ScenarioStat {

frontend/web/src/components/RunList.tsx

@@ -230,7 +230,7 @@ function RunRow({ r, selected, targetName, scenarioName, onSelect, onOpenReport,
 
   const bar = (() => {
     if (status === 'completed') {
-      const passRate = (r.summary as Record<string, unknown>)?.pass_rate as number | undefined
+      const passRate = r.summary?.pass_rate
       if (passRate != null) {
         const pct = Math.round(passRate * 100)
         const color = pct >= 80 ? statusColors.completed : pct >= 50 ? '#faad14' : statusColors.failed

frontend/web/src/pages/Campaigns.tsx

@@ -285,7 +285,7 @@ export default function CampaignsPage() {
     {
       title: '通过率', key: 'pass_rate',
       render: (_: unknown, r: Run) => {
-        const rate = (r.summary as { pass_rate?: number } | null)?.pass_rate
+        const rate = r.summary?.pass_rate
         return rate == null ? '—' : fmtPct(rate)
       },
     },

frontend/web/src/pages/Home.tsx

@@ -198,7 +198,7 @@ export default function HomePage() {
 }
 
 function RecentRunRow({ run, onOpen }: { run: Run; onOpen: () => void }) {
-  const rate = (run.summary as Record<string, unknown>)?.pass_rate as number | undefined
+  const rate = run.summary?.pass_rate
   const dotColor = statusColors[run.status] ?? colors.textMuted
   const trigger = run.triggered_by ?? 'manual'
   return (

frontend/web/src/pages/Reports.tsx

@@ -188,7 +188,7 @@ export default function ReportsPage() {
   )
 
   const buildOption = (r: Run) => {
-    const passRate = (r.summary as Record<string, unknown>)?.pass_rate as number | undefined
+    const passRate = r.summary?.pass_rate
     const pct = passRate != null ? `${Math.round(passRate * 100)}%` : '-'
     const scenario = r.scenario_name || r.scenario_id.slice(0, 8)
     const version = `v${r.scenario_version ?? 1}`

frontend/web/src/pages/Runs.tsx

@@ -287,12 +287,12 @@ function DetailWorkspace({
     )
   }
 
-  const summary = (run.summary || {}) as Record<string, unknown>
+  const summary = run.summary
-  const passRate = summary.pass_rate as number | undefined
+  const passRate = summary?.pass_rate ?? undefined
-  const totalCases = summary.total_cases as number | undefined
+  const totalCases = summary?.total_cases
-  const passedCases = summary.passed_cases as number | undefined
+  const passedCases = summary?.passed_cases
-  const totalRules = summary.total_rules as number | undefined
+  const totalRules = summary?.total_rules
-  const passedRules = summary.passed_rules as number | undefined
+  const passedRules = summary?.passed_rules
 
   const activeKey = tabOverrideId === run.id && tabKey ? tabKey : autoTab
 

tests/integration/test_campaign_runner.py

@@ -87,7 +87,7 @@ async def test_advance_spawns_due_runs_matching_plan(seeded_db):
         assert run.scenario_id == "s-1"
         assert run.triggered_by == RunTrigger.CAMPAIGN
         assert run.status == RunStatus.COMPLETED
-        assert run.summary["total_cases"] == 1
+        assert run.summary.total_cases == 1
 
 
 async def test_advance_is_idempotent(seeded_db):

tests/integration/test_stats_api.py

@@ -104,3 +104,28 @@ def test_trend_returns_daily_points(client_with_db):
     assert len(points) == 1
     assert points[0]["run_count"] == 2
     assert points[0]["pass_rate"] == pytest.approx(75.0)
+
+
+def test_dashboard_follows_adr_0004(client_with_db):
+    """故障 run 计 0.0 进分母；用户取消的 run 整体排除（ADR-0004）。"""
+    client, session = client_with_db
+    _seed(session)  # two completed runs: 1.0 and 0.5
+    repo = RunRepository(session)
+    target_id = repo.list_all()[0].target_id
+    scenario_id = repo.list_all()[0].scenario_id
+
+    faulted = repo.create(EvalRun(
+        target_id=target_id, scenario_id=scenario_id, status=RunStatus.FAILED,
+    ))
+    faulted.summary = {"error": "channel exploded"}
+    repo.update(faulted)
+
+    cancelled = repo.create(EvalRun(
+        target_id=target_id, scenario_id=scenario_id, status=RunStatus.FAILED,
+    ))
+    cancelled.summary = {"error": {"code": "cancelled_by_user", "message": "stop"}}
+    repo.update(cancelled)
+
+    data = client.get("/api/stats/dashboard").json()
+    # (1.0 + 0.5 + 0.0[fault]) / 3 — cancelled run out of the denominator
+    assert data["overall_pass_rate"] == pytest.approx(0.5)

tests/unit/test_engine.py

@@ -12,7 +12,7 @@ import pytest
 from agenteval.channels.base import SendResult
 from agenteval.evaluation.engine import CancelledError, EvalEngine, TimeoutConfig
 from agenteval.models import (
-    Case, CaseType, ChannelType, EvalTarget, Expectation, PlatformType,
+    Case, CaseOutcomeSummary, CaseType, ChannelType, EvalTarget, Expectation, PlatformType,
     RunStatus, Scenario, TargetStatus,
 )
 from agenteval.storage.repository import RunRepository
@@ -75,8 +75,8 @@ async def test_run_single_case_completes(db_session):
     run = await engine.run()
 
     assert run.status == RunStatus.COMPLETED
-    assert run.summary["total_cases"] == 1
+    assert run.summary.total_cases == 1
-    assert run.summary["passed_cases"] == 1
+    assert run.summary.passed_cases == 1
     assert channel.send_calls == 1
     assert channel.poll_calls == 1
 
@@ -113,7 +113,7 @@ async def test_run_multiple_cases(db_session):
     run = await engine.run()
 
     assert run.status == RunStatus.COMPLETED
-    assert run.summary["total_cases"] == 3
+    assert run.summary.total_cases == 3
     assert channel.sent == ["one", "two", "three"]
 
 
@@ -174,7 +174,7 @@ async def test_cancel_before_run_marks_failed(db_session):
     run = await engine.run()
 
     assert run.status == RunStatus.FAILED
-    assert run.summary["error"]["code"] == "cancelled_by_user"
+    assert run.summary.error.code == "cancelled_by_user"
     # Engine must NOT have called the channel.
     assert channel.send_calls == 0
 
@@ -206,7 +206,7 @@ async def test_cancel_mid_run_stops_after_current_case(db_session):
     )
 
     assert run.status == RunStatus.FAILED
-    assert run.summary["error"]["code"] == "cancelled_by_user"
+    assert run.summary.error.code == "cancelled_by_user"
     # At least one case ran, but not all three.
     assert 1 <= channel.send_calls < 3
 
@@ -302,8 +302,8 @@ async def test_send_failure_aborts_case(db_session):
     run = await engine.run()
 
     assert run.status == RunStatus.COMPLETED
-    assert run.summary["failed_cases"] == 1
+    assert run.summary.failed_cases == 1
-    assert run.summary["passed_cases"] == 1
+    assert run.summary.passed_cases == 1
 
 
 # ── dynamic case generation failure ──────────────────────────────────────
@@ -323,12 +323,12 @@ async def test_dynamic_generation_failure_records_case_error(db_session):
     run = await engine.run()
 
     assert run.status == RunStatus.COMPLETED
-    assert run.summary["failed_cases"] == 1
+    assert run.summary.failed_cases == 1
-    assert run.summary["total_rules"] == 0
+    assert run.summary.total_rules == 0
     # 关键：失败原因被持久化到 summary，可在报告 / DB 查看
-    assert "case_errors" in run.summary
+    assert run.summary.case_errors
-    assert run.summary["case_errors"][0]["case_id"] == "dyn-1"
+    assert run.summary.case_errors[0]["case_id"] == "dyn-1"
-    assert "llm_config" in run.summary["case_errors"][0]["error"]
+    assert "llm_config" in run.summary.case_errors[0]["error"]
     # 被测通道不应被调用（生成阶段就失败了）
     assert channel.send_calls == 0
 
@@ -367,7 +367,7 @@ async def test_expectation_fails_case_even_when_rules_pass(db_session):
     run = await engine.run()
 
     assert run.status == RunStatus.COMPLETED
-    assert run.summary["failed_cases"] == 1
+    assert run.summary.failed_cases == 1
 
 
 async def test_expectation_and_rules_both_pass(db_session):
@@ -381,7 +381,7 @@ async def test_expectation_and_rules_both_pass(db_session):
 
     run = await engine.run()
 
-    assert run.summary["passed_cases"] == 1
+    assert run.summary.passed_cases == 1
     results = RunRepository(db_session).get_results(run.id)
     # 1 显式规则 + 2 期望派生（keyword + response_time）
     assert len(results) == 3
@@ -402,7 +402,7 @@ async def test_implicit_expectation_not_in_any_combination(db_session):
 
     run = await engine.run()
 
-    assert run.summary["failed_cases"] == 1
+    assert run.summary.failed_cases == 1
 
 
 async def test_implicit_expectation_is_hard_constraint_over_weighted(db_session):
@@ -418,7 +418,7 @@ async def test_implicit_expectation_is_hard_constraint_over_weighted(db_session)
 
     run = await engine.run()
 
-    assert run.summary["failed_cases"] == 1
+    assert run.summary.failed_cases == 1
 
 
 async def test_pure_expectation_case_behavior_unchanged(db_session):
@@ -434,8 +434,8 @@ async def test_pure_expectation_case_behavior_unchanged(db_session):
 
     run = await engine.run()
 
-    assert run.summary["passed_cases"] == 1
+    assert run.summary.passed_cases == 1
-    assert run.summary["failed_cases"] == 1
+    assert run.summary.failed_cases == 1
 
 
 # ── scenario_version snapshot (ticket 04) ────────────────────────────────
@@ -472,11 +472,11 @@ async def test_summary_contains_case_outcomes_and_case_level_pass_rate(db_sessio
 
     run = await engine.run()
 
-    outcomes = run.summary["case_outcomes"]
+    outcomes = run.summary.case_outcomes
-    assert outcomes["conn"] == {"passed": True, "connectivity": True}
+    assert outcomes["conn"] == CaseOutcomeSummary(passed=True, connectivity=True)
-    assert outcomes["bad"] == {"passed": False, "connectivity": False}
+    assert outcomes["bad"] == CaseOutcomeSummary(passed=False, connectivity=False)
     # 通过率为用例级口径（CONTEXT.md），不再是规则级
-    assert run.summary["pass_rate"] == 0.5
+    assert run.summary.pass_rate == 0.5
 
 
 async def test_connectivity_case_without_reply_fails(db_session):
@@ -492,5 +492,5 @@ async def test_connectivity_case_without_reply_fails(db_session):
 
     run = await engine.run()
 
-    assert run.summary["passed_cases"] == 0
+    assert run.summary.passed_cases == 0
-    assert run.summary["case_outcomes"]["conn"] == {"passed": False, "connectivity": False}
+    assert run.summary.case_outcomes["conn"] == CaseOutcomeSummary(passed=False, connectivity=False)

tests/unit/test_metrics.py

@@ -0,0 +1,129 @@
+"""Unit tests for the RunSummary value type and the single cross-run
+aggregation seam (ADR-0004: faults count 0.0, user-cancelled excluded)."""
+
+from agenteval.evaluation.metrics import aggregate_runs
+from agenteval.models import EvalRun, RunStatus, RunSummary
+
+# ── RunSummary parsing (legacy dict rows must keep parsing) ────────────────
+
+
+def test_parses_full_engine_summary():
+    s = RunSummary.model_validate({
+        "total_cases": 3,
+        "passed_cases": 2,
+        "failed_cases": 1,
+        "total_rules": 5,
+        "passed_rules": 4,
+        "pass_rate": 0.6667,
+        "avg_latency_ms": 123.4,
+        "case_outcomes": {"c1": {"passed": True, "connectivity": False}},
+        "case_errors": [{"case_id": "c2", "stage": "generate_messages", "error": "boom"}],
+        "model_configs": {"judge": {"model": "m"}},
+    })
+    assert s.pass_rate == 0.6667
+    assert s.avg_latency_ms == 123.4
+    assert s.case_outcomes["c1"].passed is True
+    assert s.case_outcomes["c1"].connectivity is False
+    assert s.error is None
+
+
+def test_parses_legacy_string_error():
+    s = RunSummary.model_validate({"error": "channel exploded"})
+    assert s.error is not None
+    assert s.error.code == "error"
+    assert s.error.message == "channel exploded"
+    assert s.is_cancelled is False
+
+
+def test_parses_object_error_and_detects_cancel():
+    s = RunSummary.model_validate({"error": {"code": "cancelled_by_user", "message": "评测已手动停止"}})
+    assert s.error is not None
+    assert s.error.code == "cancelled_by_user"
+    assert s.is_cancelled is True
+
+
+def test_missing_keys_default_and_unknown_keys_tolerated():
+    s = RunSummary.model_validate({"pass_rate": 0.5, "some_future_key": 1})
+    assert s.pass_rate == 0.5
+    assert s.total_cases == 0
+    assert s.case_outcomes == {}
+    assert s.error is None
+
+
+def test_eval_run_summary_field_is_typed():
+    run = EvalRun(target_id="t", scenario_id="s", summary={"pass_rate": 1.0})
+    assert isinstance(run.summary, RunSummary)
+    assert run.summary.pass_rate == 1.0
+
+
+# ── aggregate_runs (ADR-0004) ──────────────────────────────────────────────
+
+
+def _run(status, *, pass_rate=None, latency=None, error=None) -> EvalRun:
+    summary = None
+    if pass_rate is not None or latency is not None or error is not None:
+        summary = {}
+        if pass_rate is not None:
+            summary["pass_rate"] = pass_rate
+        if latency is not None:
+            summary["avg_latency_ms"] = latency
+        if error is not None:
+            summary["error"] = error
+    return EvalRun(target_id="t", scenario_id="s", status=status, summary=summary)
+
+
+def test_empty_runs_aggregate_to_none():
+    agg = aggregate_runs([])
+    assert agg == {"run_count": 0, "pass_rate": None, "availability": None, "avg_latency_ms": None}
+
+
+def test_completed_runs_average_pass_rate_and_latency():
+    agg = aggregate_runs([
+        _run(RunStatus.COMPLETED, pass_rate=1.0, latency=100),
+        _run(RunStatus.COMPLETED, pass_rate=0.5, latency=200),
+    ])
+    assert agg["run_count"] == 2
+    assert agg["pass_rate"] == 0.75
+    assert agg["availability"] == 1.0
+    assert agg["avg_latency_ms"] == 150.0
+
+
+def test_faulted_run_counts_zero_in_both_denominators():
+    # ADR-0004: a genuine execution fault drags pass_rate AND availability down.
+    agg = aggregate_runs([
+        _run(RunStatus.COMPLETED, pass_rate=1.0),
+        _run(RunStatus.FAILED, error="channel exploded"),
+    ])
+    assert agg["pass_rate"] == 0.5
+    assert agg["availability"] == 0.5
+
+
+def test_cancelled_run_excluded_from_denominators():
+    # ADR-0004: user cancellation is neither a quality nor availability signal.
+    agg = aggregate_runs([
+        _run(RunStatus.COMPLETED, pass_rate=1.0),
+        _run(RunStatus.FAILED, error={"code": "cancelled_by_user", "message": "stop"}),
+    ])
+    assert agg["run_count"] == 2          # what happened stays visible
+    assert agg["pass_rate"] == 1.0        # cancelled run out of the denominator
+    assert agg["availability"] == 1.0
+
+
+def test_all_cancelled_aggregates_to_none():
+    agg = aggregate_runs([
+        _run(RunStatus.FAILED, error={"code": "cancelled_by_user", "message": "stop"}),
+    ])
+    assert agg["run_count"] == 1
+    assert agg["pass_rate"] is None
+    assert agg["availability"] is None
+    assert agg["avg_latency_ms"] is None
+
+
+def test_in_flight_runs_count_zero_not_cancelled():
+    # pending/running (no summary) are not cancelled — they count 0.0 for now.
+    agg = aggregate_runs([
+        _run(RunStatus.RUNNING),
+        _run(RunStatus.PENDING),
+    ])
+    assert agg["pass_rate"] == 0.0
+    assert agg["availability"] == 0.0

tests/unit/test_model_runtime_integration.py

@@ -57,7 +57,7 @@ async def test_dynamic_case_uses_generator_binding_and_records_snapshot(db_sessi
 
     assert gateway.chat_calls == 1
     assert engine.channel.sent == ["问题一", "问题二"]
-    snapshot = run.summary["model_configs"]["generator"]
+    snapshot = run.summary.model_configs["generator"]
     assert snapshot["id"] == config_id
     assert snapshot["model_name"] == "generator-v1"
     assert "api_key" not in snapshot

tests/unit/test_orphan_runs.py

@@ -24,7 +24,7 @@ def test_mark_orphans_failed(db_session):
     for rid in (running_id, pending_id):
         run = repo.get(rid)
         assert run.status == RunStatus.FAILED
-        assert run.summary["error"]["code"] == "interrupted"
+        assert run.summary.error.code == "interrupted"
         assert run.completed_at is not None
     # 已完结的运行不受影响
     assert repo.get(completed_id).status == RunStatus.COMPLETED

tests/unit/test_report.py

@@ -391,7 +391,7 @@ def test_authoritative_case_outcomes_override_reconstruction(report_session):
     repo = RunRepository(report_session)
     run = repo.get(run_id)
     # 模拟 weighted 阈值未达：规则单条通过但用例判失败（反推 all() 会误判 True）
-    run.summary = {**run.summary, "case_outcomes": {"c0": {"passed": False, "connectivity": False}}}
+    run.summary = {**run.summary.model_dump(), "case_outcomes": {"c0": {"passed": False, "connectivity": False}}}
     repo.update(run)
 
     report = generate_report(run_id, report_session)

tests/unit/test_s2_rules_and_logic.py

@@ -312,7 +312,7 @@ async def test_rule_logic_all_passes_when_all_pass(db_session):
     engine = _build_engine(scenario, db_session)
     run = await engine.run()
     assert run.status == RunStatus.COMPLETED
-    assert run.summary["passed_cases"] == 1
+    assert run.summary.passed_cases == 1
 
 
 async def test_rule_logic_all_fails_when_one_fails(db_session):
@@ -327,7 +327,7 @@ async def test_rule_logic_all_fails_when_one_fails(db_session):
     engine = _build_engine(scenario, db_session)
     run = await engine.run()
     assert run.status == RunStatus.COMPLETED
-    assert run.summary["failed_cases"] == 1
+    assert run.summary.failed_cases == 1
 
 
 async def test_rule_logic_any_passes_when_one_passes(db_session):
@@ -342,7 +342,7 @@ async def test_rule_logic_any_passes_when_one_passes(db_session):
     engine = _build_engine(scenario, db_session)
     run = await engine.run()
     assert run.status == RunStatus.COMPLETED
-    assert run.summary["passed_cases"] == 1
+    assert run.summary.passed_cases == 1
 
 
 async def test_rule_logic_weighted_passes_above_threshold(db_session):
@@ -358,7 +358,7 @@ async def test_rule_logic_weighted_passes_above_threshold(db_session):
     engine = _build_engine(scenario, db_session)
     run = await engine.run()
     assert run.status == RunStatus.COMPLETED
-    assert run.summary["passed_cases"] == 1
+    assert run.summary.passed_cases == 1
 
 
 async def test_rule_logic_weighted_fails_below_threshold(db_session):
@@ -374,4 +374,4 @@ async def test_rule_logic_weighted_fails_below_threshold(db_session):
     engine = _build_engine(scenario, db_session)
     run = await engine.run()
     assert run.status == RunStatus.COMPLETED
-    assert run.summary["failed_cases"] == 1
+    assert run.summary.failed_cases == 1