solahqb/AgentEvalTool
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request 'feat(report): 上线验收报告 go/no-go 结论' (#28) from feat/go-no-go-verdict into main
All checks were successful
CI / test (push) Successful in 4m6s
Details

Browse Source
This commit is contained in:
solahqb 2026-08-25 06:31:42 +00:00
commit 2f08e7bf06
4 changed files with 662 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
backend/agenteval/evaluation/go_no_go.py Normal file
View File

@ -0,0 +1,137 @@
"""Go/No-Go acceptance verdict for evaluation runs."""
from datetime import datetime, timezone
from typing import Any, Optional
from pydantic import BaseModel, Field
class AcceptanceCriteria(BaseModel):
"""Acceptance criteria for go/no-go verdict."""
judged_pass_rate_min: float = Field(default=0.95, description="Minimum judged pass rate (0-1)")
pass_rate_min: float = Field(default=0.90, description="Minimum overall pass rate (0-1)")
avg_latency_max_ms: Optional[float] = Field(default=None, description="Maximum average latency in ms")
availability_min: Optional[float] = Field(default=None, description="Minimum availability (0-1), for campaign level")
class CriterionResult(BaseModel):
"""Result of checking a single criterion."""
criterion: str = Field(description="Criterion name")
threshold: float = Field(description="Threshold value")
actual: float = Field(description="Actual value")
passed: bool = Field(description="Whether the criterion passed")
detail: str = Field(default="", description="Human-readable detail")
class GoNoGoVerdict(BaseModel):
"""Go/No-Go verdict for an evaluation run."""
decision: str = Field(description="go | no_go | conditional")
summary: str = Field(description="Human-readable summary")
criteria_results: list[CriterionResult] = Field(default_factory=list)
generated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
def evaluate_go_no_go(
summary: dict[str, Any],
criteria: AcceptanceCriteria | None = None,
) -> GoNoGoVerdict:
"""Evaluate go/no-go verdict based on run summary and acceptance criteria.
Args:
summary: Run summary dict with keys like judged_pass_rate, pass_rate, avg_latency_ms
criteria: Acceptance criteria. If None, uses defaults.
Returns:
GoNoGoVerdict with decision, summary, and criteria results.
"""
if criteria is None:
criteria = AcceptanceCriteria()
results: list[CriterionResult] = []
# Check judged pass rate (only if present in summary)
judged_pass_rate = summary.get("judged_pass_rate")
if judged_pass_rate is None:
judged_pass_rate = summary.get("pass_rate")
if judged_pass_rate is not None:
passed = judged_pass_rate >= criteria.judged_pass_rate_min
results.append(CriterionResult(
criterion="judged_pass_rate",
threshold=criteria.judged_pass_rate_min,
actual=judged_pass_rate,
passed=passed,
detail=f"判定型通过率 {judged_pass_rate*100:.1f}% {'>=' if passed else '<'} {criteria.judged_pass_rate_min*100:.0f}%",
))
# Check overall pass rate (only if different from judged and present)
pass_rate = summary.get("pass_rate")
if pass_rate is not None and pass_rate != judged_pass_rate:
passed = pass_rate >= criteria.pass_rate_min
results.append(CriterionResult(
criterion="pass_rate",
threshold=criteria.pass_rate_min,
actual=pass_rate,
passed=passed,
detail=f"全量通过率 {pass_rate*100:.1f}% {'>=' if passed else '<'} {criteria.pass_rate_min*100:.0f}%",
))
# Check average latency
avg_latency_ms = summary.get("avg_latency_ms")
if avg_latency_ms is not None and criteria.avg_latency_max_ms is not None:
passed = avg_latency_ms <= criteria.avg_latency_max_ms
results.append(CriterionResult(
criterion="avg_latency_ms",
threshold=criteria.avg_latency_max_ms,
actual=avg_latency_ms,
passed=passed,
detail=f"平均延迟 {avg_latency_ms:.0f}ms {'<=' if passed else '>'} {criteria.avg_latency_max_ms:.0f}ms",
))
# Check availability (for campaign level)
availability = summary.get("overall_availability") or summary.get("availability")
if availability is not None and criteria.availability_min is not None:
passed = availability >= criteria.availability_min
results.append(CriterionResult(
criterion="availability",
threshold=criteria.availability_min,
actual=availability,
passed=passed,
detail=f"可用性 {availability*100:.1f}% {'>=' if passed else '<'} {criteria.availability_min*100:.0f}%",
))
# Determine overall decision
if not results:
return GoNoGoVerdict(
decision="conditional",
summary="无可用指标进行评估",
criteria_results=[],
)
all_passed = all(r.passed for r in results)
if all_passed:
decision = "go"
main_metric = results[0]
summary_text = f"通过率 {main_metric.actual*100:.0f}%，达标，建议上线"
else:
# Check if core metrics (pass rate) failed
core_failed = any(
not r.passed and r.criterion in ("judged_pass_rate", "pass_rate", "availability")
for r in results
)
if core_failed:
decision = "no_go"
failed_metrics = [r for r in results if not r.passed]
summary_text = f"核心指标未达标（{', '.join(r.criterion for r in failed_metrics)}），不建议上线"
else:
decision = "conditional"
risky_metrics = [r for r in results if not r.passed]
summary_text = f"部分指标达标，存在风险项（{', '.join(r.criterion for r in risky_metrics)}），建议修复后复测"
return GoNoGoVerdict(
decision=decision,
summary=summary_text,
criteria_results=results,
)

28
backend/agenteval/evaluation/report.py
View File

@ -12,6 +12,7 @@ from typing import Any, Optional
from sqlmodel import Session
from agenteval.evaluation.case_verdict import build_case_evidence, resolve_case_verdicts
from agenteval.evaluation.go_no_go import evaluate_go_no_go
from agenteval.evaluation.metrics import aggregate_runs
from agenteval.evaluation.report_render import render_html, render_json, render_markdown
from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary
@ -103,6 +104,21 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
if judged_pass_rate is None and judged_total > 0:
judged_pass_rate = round((passed_cases - connectivity_count) / judged_total, 4)
summary_dict = {
"total_cases": total_cases,
"passed_cases": passed_cases,
"failed_cases": summary.failed_cases,
"total_rules": summary.total_rules,
"passed_rules": summary.passed_rules,
"pass_rate": summary.pass_rate if summary.pass_rate is not None else 0.0,
"connectivity_cases": connectivity_count,
"judged_pass_rate": judged_pass_rate,
"avg_latency_ms": summary.avg_latency_ms,
}
# Generate go/no-go verdict
verdict = evaluate_go_no_go(summary_dict)
return {
"run_id": run.id,
"target_id": run.target_id,
@ -114,16 +130,8 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
"status": run.status.value,
"started_at": iso_utc(run.started_at),
"completed_at": iso_utc(run.completed_at),
"summary": {
"total_cases": total_cases,
"passed_cases": passed_cases,
"failed_cases": summary.failed_cases,
"total_rules": summary.total_rules,
"passed_rules": summary.passed_rules,
"pass_rate": summary.pass_rate if summary.pass_rate is not None else 0.0,
"connectivity_cases": connectivity_count,
"judged_pass_rate": judged_pass_rate,
},
"summary": summary_dict,
"go_no_go": verdict.model_dump(mode="json"),
"cases": cases,
}

369
research/go-no-go-report.md Normal file
View File

@ -0,0 +1,369 @@
# Go/No-Go 验收报告技术方案
> Ticket: [#20](https://git.solahqb22.cn/solahqb/AgentEvalTool/issues/20)
> Wayfinder: [#13](https://git.solahqb22.cn/solahqb/AgentEvalTool/issues/13)
> 日期: 2026-07-15
---
## 1. 现有报告能力分析
### 1.1 报告体系概览
系统当前有三套独立的报告生成管线：
| 报告类型 | 生成入口 | 渲染输出 | API 端点 |
|---------|---------|---------|---------|
| **单次运行报告** | `evaluation/report.py::generate_report()` | HTML / JSON / Markdown | `GET /api/reports/{run_id}` |
| **活动周期报告** | `evaluation/report.py::generate_campaign_report()` | JSON dict + Markdown | `GET /api/campaigns/{id}/report` |
| **智能评估报告** | `intelligent_eval/report.py::render_report_markdown()` | JSON + Markdown | `GET /api/intelligent-evals/{id}/report` |
### 1.2 单次运行报告（Run Report）
**数据源**：`RunSummary` 模型（`models.py`），由 `build_run_summary()` 在引擎执行完毕后一次性写入。
**已有指标**：
- `total_cases` / `passed_cases` / `failed_cases` -- 用例级计数
- `pass_rate` -- 含连通用例的全量通过率
- `judged_pass_rate` -- 剔除连通用例后的判定型通过率
- `total_rules` / `passed_rules` -- 规则级计数
- `avg_latency_ms` -- 平均延迟
- `case_outcomes` -- 每个用例的 passed/connectivity 权威判定
**结论性判断现状**：**无**。报告只呈现原始数字，不做任何阈值比对。HTML 模板只有四个统计卡片（用例总数、通过用例、规则总数、规则通过率），没有"达标/不达标"标记。Markdown 输出同理，只列数据表格。
### 1.3 活动周期报告（Campaign Report）
**已有指标**：
- `overall_pass_rate` / `overall_availability` / `avg_latency_ms` -- 整窗聚合
- `time_trend` -- 12 时段分桶的通过率/可用性/时延趋势
- `capability_summary` -- 按场景分组的指标汇总
- 智能分析（LLM 诊断）：总体结论 + 问题清单 + 改善建议
- 周期对比：基线 vs 本期的指标 delta + 趋势叙述
**结论性判断现状**：**部分**。智能分析的 `overall` 字段包含 LLM 生成的叙述性结论（如"整体表现稳定，建议关注场景 X 的退化"），但这是自然语言判断，不是机械的 go/no-go 判定。没有可配置的阈值比对逻辑。
### 1.4 智能评估报告（Intelligent Eval Report）
**已有指标**：
- `scores.overall` + `scores.dimensions` -- 维度评分
- `findings` -- 问题发现（含 severity/dimension/evidence）
- `priority_recommendations` -- 改进建议
**结论性判断现状**：**无**。有评分但无阈值判定。
### 1.5 已有的阈值机制（可复用）
| 层级 | 阈值 | 位置 | 用途 |
|------|------|------|------|
| 用例级 | `rule_pass_threshold` (默认 0.6) | `Case` 模型 | WEIGHTED 规则逻辑的加权分通过线 |
| 规则级 | `max_ms` | `ResponseTimeRule` | 单轮延迟上限 |
| 规则级 | `keywords_include/exclude` | `KeywordMatchRule` | 关键词匹配 |
| 用例级 | `coherence_min_score` | `Expectation` | 连贯性最低分（隐式规则） |
**关键发现**：系统已有完善的**用例级**判定链路（`judgement.combine_case_outcome` -> `build_run_summary` -> `case_outcomes`），但**完全缺少运行级/活动级的阈值判定**。go/no-go 需要填补的正是这个空白。
---
## 2. Go/No-Go 结论的实现方案
### 2.1 核心设计
新增一个纯函数模块 `evaluation/go_no_go.py`，职责单一：接收报告 dict + 阈值配置 -> 输出结构化结论。
```python
# 提议的数据模型
class AcceptanceCriteria(BaseModel):
"""上线验收标准"""
judged_pass_rate_min: float = 0.95 # 判定型通过率下限
pass_rate_min: float = 0.90 # 全量通过率下限
avg_latency_max_ms: Optional[float] = None # 平均延迟上限
availability_min: Optional[float] = None # 可用性下限（活动级）
no_high_severity_finding: bool = False # 不允许有高严重度发现（智能评估）
class GoNoGoVerdict(BaseModel):
"""Go/No-Go 结论"""
decision: str # "go" | "no_go" | "conditional"
summary: str # 人类可读结论，如"通过率 95%，达标，建议上线"
criteria_results: list[CriterionResult]
generated_at: datetime
class CriterionResult(BaseModel):
"""单条标准的比对结果"""
criterion: str # 标准名称
threshold: float # 阈值
actual: float # 实际值
passed: bool # 是否达标
detail: str # 说明
```
### 2.2 判定逻辑
```
function evaluate_go_no_go(report_dict, criteria):
results = []
// 1. 判定型通过率
if criteria.judged_pass_rate_min is set:
actual = report.summary.judged_pass_rate ?? report.summary.pass_rate
results.append(CriterionResult(
criterion="judged_pass_rate",
threshold=criteria.judged_pass_rate_min,
actual=actual,
passed=actual >= criteria.judged_pass_rate_min
))
// 2. 全量通过率
if criteria.pass_rate_min is set:
actual = report.summary.pass_rate
results.append(...)
// 3. 平均延迟
if criteria.avg_latency_max_ms is set:
actual = report.summary.avg_latency_ms
results.append(CriterionResult(
criterion="avg_latency",
threshold=criteria.avg_latency_max_ms,
actual=actual,
passed=actual <= criteria.avg_latency_max_ms
))
// 4. 可用性（活动级）
if criteria.availability_min is set:
actual = report.summary.overall_availability
results.append(...)
// 综合判定
all_passed = all(r.passed for r in results)
if all_passed:
decision = "go"
summary = f"通过率 {actual*100:.0f}%，达标，建议上线"
elif any critical failures:
decision = "no_go"
summary = f"通过率 {actual*100:.0f}%，未达标（阈值 {threshold*100:.0f}%），不建议上线"
else:
decision = "conditional"
summary = "部分指标达标，存在风险项，建议修复后复测"
return GoNoGoVerdict(decision, summary, results)
```
### 2.3 三级结论语义
| 结论 | 条件 | 含义 |
|------|------|------|
| **go** | 全部指标达标 | 建议上线 |
| **no_go** | 核心指标（通过率/可用性）未达标 | 不建议上线 |
| **conditional** | 核心达标但非核心指标（延迟等）有风险 | 建议修复后复测 |
### 2.4 报告渲染集成
**HTML 报告**：在 summary 卡片区域上方新增一个醒目的结论横幅：
- go: 绿色背景 "GO - 建议上线"
- no_go: 红色背景 "NO-GO - 不建议上线"
- conditional: 黄色背景 "CONDITIONAL - 存在风险"
下方增加一个"验收标准比对表"，逐条列出阈值、实际值、是否达标。
**Markdown 报告**：在汇总表格后增加：
```markdown
## 上线验收结论
**结论**: GO / NO-GO / CONDITIONAL
| 指标 | 阈值 | 实际值 | 结果 |
|------|------|--------|------|
| 判定型通过率 | >= 95% | 96.2% | PASS |
| 平均延迟 | <= 5000ms | 3200ms | PASS |
```
**JSON 报告**：在顶层新增 `go_no_go` 字段，包含完整的 `GoNoGoVerdict` 结构。
### 2.5 API 设计
**方案 A（推荐）**：在现有报告端点中自动附带
```
GET /api/reports/{run_id}
-> { ...existing report..., go_no_go: { decision, summary, criteria_results } }
```
优点：前端无需改动调用逻辑，结论随报告自动返回。
缺点：需要知道使用哪套阈值 -> 从 Scenario 或全局配置读取。
**方案 B**：独立端点
```
GET /api/reports/{run_id}/verdict?pass_rate_min=0.95&latency_max=5000
```
优点：阈值灵活，可按需传入。
缺点：增加前端调用复杂度。
**推荐方案 A**，因为阈值应该在创建评测时就确定（绑定到 Scenario 或全局配置），而非每次查看报告时指定。
---
## 3. 达标阈值的配置方式
### 3.1 配置层级设计
```
全局默认（Settings）
└── 场景级覆盖（Scenario.acceptance_criteria）
└── 运行级覆盖（EvalRun.acceptance_criteria，可选）
```
**优先级**：运行级 > 场景级 > 全局默认
### 3.2 具体实现位置
| 层级 | 存储位置 | 配置方式 |
|------|---------|---------|
| **全局默认** | `config/settings.py` 新增字段 | 环境变量 `AGENTEVAL_DEFAULT_PASS_RATE_MIN=0.95` |
| **场景级** | `Scenario` 模型新增 `acceptance_criteria: Optional[AcceptanceCriteria]` | API/前端创建场景时配置 |
| **运行级** | `EvalRun` 模型新增 `acceptance_criteria: Optional[AcceptanceCriteria]`（或从 Scenario 继承） | 创建 run 时可选覆盖 |
### 3.3 全局默认配置示例
```python
# settings.py 新增
default_acceptance_criteria: dict[str, Any] = Field(
default_factory=lambda: {
"judged_pass_rate_min": 0.95,
"pass_rate_min": 0.90,
},
description="Default acceptance criteria for go/no-go verdicts.",
)
```
### 3.4 场景级配置示例
在 Scenario 模型中新增可选字段：
```python
class Scenario(BaseModel):
# ... existing fields ...
acceptance_criteria: Optional[dict[str, Any]] = None
# 例如: {"judged_pass_rate_min": 0.98, "avg_latency_max_ms": 3000}
```
前端在场景编辑页增加"验收标准"配置区域（折叠面板），默认展示全局默认值，用户可覆盖。
### 3.5 数据库迁移
需要 Alembic 迁移脚本为 `scenarios` 表和 `eval_runs` 表添加 `acceptance_criteria` JSON 列。SQLite 的 `render_as_batch=True` 已配置，迁移无特殊障碍。
---
## 4. 工作量估算
### 4.1 后端
| 任务 | 人天 | 说明 |
|------|------|------|
| `evaluation/go_no_go.py` 核心判定逻辑 | 0.5 | 纯函数，单测覆盖 |
| `AcceptanceCriteria` / `GoNoGoVerdict` 数据模型 | 0.5 | Pydantic 模型 |
| `RunSummary` / `Scenario` 模型扩展 | 0.5 | 新增字段 + 兼容处理 |
| `generate_report()` 集成 go_no_go 结论 | 0.5 | 在报告 dict 中附加 verdict |
| `report_render.py` 三种格式渲染 | 1.0 | HTML 横幅 + Markdown 表格 + JSON 字段 |
| `settings.py` 全局默认配置 | 0.25 | 新增 Settings 字段 |
| Alembic 迁移 | 0.25 | scenarios + eval_runs 加列 |
| API 端点调整 | 0.5 | reports router 附带 verdict |
| 单元测试 + 集成测试 | 1.0 | go_no_go 逻辑 + API 覆盖 |
| **后端小计** | **5.0** | |
### 4.2 前端
| 任务 | 人天 | 说明 |
|------|------|------|
| 场景编辑页增加"验收标准"配置 | 1.0 | FormDrawer 内新增折叠面板 |
| Run 报告详情页展示 go/no-go 结论 | 1.0 | 结论横幅 + 比对表格 |
| 活动报告页展示活动级结论 | 0.5 | 复用组件 |
| **前端小计** | **2.5** | |
### 4.3 总计
| 模块 | 人天 |
|------|------|
| 后端 | 5.0 |
| 前端 | 2.5 |
| 联调 + 部署 | 0.5 |
| **总计** | **8.0 人天** |
如果只做后端（API 返回 verdict，前端后续迭代），可压缩到 **5.0 人天**。
---
## 5. 推荐方案
### 5.1 推荐：方案 A + 三级配置 + 渐进式交付
**理由**：
1. **纯函数核心**（`go_no_go.py`）：与现有架构一致（`metrics.py` / `run_summary.py` 都是纯函数），易测试、易扩展。
2. **三级阈值配置**（全局默认 -> 场景级 -> 运行级）：
- 全局默认保证开箱即用，不需要每个场景都配置
- 场景级覆盖满足差异化需求（如关键场景要求 98% 通过率）
- 运行级覆盖保留灵活性（如临时加严测试）
3. **自动附带而非独立端点**：前端零改动即可获得 verdict 数据，降低集成成本。
4. **三级结论（go/no_go/conditional）**：比二元判定更实用。"conditional" 覆盖了"通过率达标但延迟偏高"这类常见场景，避免误判。
5. **复用现有指标**：不需要新增数据采集，`judged_pass_rate` / `pass_rate` / `avg_latency_ms` / `availability` 均已由引擎计算并持久化。go/no-go 只是在读路径上增加一层阈值比对。
### 5.2 实施路径
```
Phase 1（MVP，5 人天）:
- go_no_go.py 核心逻辑
- 全局默认阈值（Settings）
- generate_report() 集成
- JSON/Markdown 渲染
- 单元测试
Phase 2（完善，3 人天）:
- 场景级 acceptance_criteria 配置
- HTML 渲染（结论横幅）
- 前端场景编辑页配置面板
- 前端报告展示页结论展示
Phase 3（可选增强）:
- 活动级 go/no-go（跨 run 聚合判定）
- 智能评估报告的评分阈值集成
- Webhook 推送 verdict（CI/CD 集成）
```
### 5.3 关键决策点
| 决策 | 推荐 | 备选 | 理由 |
|------|------|------|------|
| 核心指标选择 | `judged_pass_rate` 为主 | `pass_rate` | 连通用例无判定意义，`judged_pass_rate` 更准确反映质量 |
| 阈值存储 | JSON 列（灵活） | 独立表（规范化） | 阈值结构简单且固定，JSON 足够，避免过度设计 |
| 结论渲染位置 | 报告顶部横幅 | 报告底部 | 结论应第一时间可见，类似体检报告的"总结" |
| 活动级 go/no-go | Phase 3 再做 | 同期实现 | 活动级需要跨 run 聚合，复杂度较高，且 ticket #20 聚焦单次验收 |
### 5.4 与现有架构的契合度
- **ADR-0002（通过率口径）**：go/no-go 直接消费 `judged_pass_rate`，口径一致
- **ADR-0004（聚合口径）**：活动级聚合复用 `aggregate_runs`，不重算
- **规则注册表模式**：go_no_go 判定器可设计为可扩展的（未来可能增加新指标）
- **纯函数渲染**：遵循 `report_render.py` 的 dict-in/string-out 模式
---
## 附录：关键代码路径
| 文件 | 职责 |
|------|------|
| `backend/agenteval/evaluation/report.py` | 报告生成（`generate_report` / `generate_campaign_report`） |
| `backend/agenteval/evaluation/report_render.py` | 报告渲染（HTML/Markdown/JSON） |
| `backend/agenteval/evaluation/run_summary.py` | 运行汇总（`build_run_summary`） |
| `backend/agenteval/evaluation/metrics.py` | 跨运行聚合（`aggregate_runs`） |
| `backend/agenteval/evaluation/judgement.py` | 用例判定（`combine_case_outcome`） |
| `backend/agenteval/evaluation/case_verdict.py` | 用例判定读路径（`resolve_case_verdicts`） |
| `backend/agenteval/models.py` | 数据模型（`RunSummary` / `Scenario` / `Case`） |
| `backend/agenteval/config/settings.py` | 全局配置 |
| `backend/agenteval/web/routers/reports.py` | 报告 API 端点 |

138
tests/unit/test_go_no_go.py Normal file
View File

@ -0,0 +1,138 @@
"""Tests for go/no-go acceptance verdict."""
import pytest
from agenteval.evaluation.go_no_go import (
AcceptanceCriteria,
CriterionResult,
GoNoGoVerdict,
evaluate_go_no_go,
)
def test_go_verdict_all_pass():
"""All criteria met -> go."""
summary = {
"judged_pass_rate": 0.96,
"pass_rate": 0.95,
"avg_latency_ms": 2000,
}
criteria = AcceptanceCriteria(
judged_pass_rate_min=0.95,
pass_rate_min=0.90,
avg_latency_max_ms=5000,
)
verdict = evaluate_go_no_go(summary, criteria)
assert verdict.decision == "go"
assert "达标" in verdict.summary
assert all(r.passed for r in verdict.criteria_results)
def test_no_go_verdict_core_failed():
"""Core metric (pass rate) failed -> no_go."""
summary = {
"judged_pass_rate": 0.80,
"pass_rate": 0.75,
}
criteria = AcceptanceCriteria(
judged_pass_rate_min=0.95,
pass_rate_min=0.90,
)
verdict = evaluate_go_no_go(summary, criteria)
assert verdict.decision == "no_go"
assert "不建议上线" in verdict.summary
assert any(not r.passed for r in verdict.criteria_results)
def test_conditional_verdict_non_core_risk():
"""Core passed but non-core (latency) failed -> conditional."""
summary = {
"judged_pass_rate": 0.96,
"pass_rate": 0.95,
"avg_latency_ms": 8000,
}
criteria = AcceptanceCriteria(
judged_pass_rate_min=0.95,
pass_rate_min=0.90,
avg_latency_max_ms=5000,
)
verdict = evaluate_go_no_go(summary, criteria)
assert verdict.decision == "conditional"
assert "风险" in verdict.summary
def test_default_criteria():
"""Default criteria should be applied when None."""
summary = {
"judged_pass_rate": 0.96,
"pass_rate": 0.95,
}
verdict = evaluate_go_no_go(summary, None)
assert verdict.decision == "go"
assert len(verdict.criteria_results) >= 1
def test_empty_summary():
"""Empty summary -> conditional with no results."""
verdict = evaluate_go_no_go({}, None)
assert verdict.decision == "conditional"
assert "无可用指标" in verdict.summary
assert len(verdict.criteria_results) == 0
def test_criterion_result_model():
"""CriterionResult model should work correctly."""
result = CriterionResult(
criterion="pass_rate",
threshold=0.95,
actual=0.96,
passed=True,
detail="通过率 96% >= 95%",
)
assert result.criterion == "pass_rate"
assert result.passed is True
def test_verdict_model():
"""GoNoGoVerdict model should work correctly."""
verdict = GoNoGoVerdict(
decision="go",
summary="测试通过",
criteria_results=[],
)
assert verdict.decision == "go"
assert verdict.generated_at is not None
def test_latency_only_when_configured():
"""Latency should only be checked when avg_latency_max_ms is set."""
summary = {
"judged_pass_rate": 0.96,
"avg_latency_ms": 10000, # High latency
}
# Without latency threshold
criteria_no_latency = AcceptanceCriteria(judged_pass_rate_min=0.95)
verdict1 = evaluate_go_no_go(summary, criteria_no_latency)
assert verdict1.decision == "go"
assert all(r.criterion != "avg_latency_ms" for r in verdict1.criteria_results)
# With latency threshold
criteria_with_latency = AcceptanceCriteria(
judged_pass_rate_min=0.95,
avg_latency_max_ms=5000,
)
verdict2 = evaluate_go_no_go(summary, criteria_with_latency)
assert verdict2.decision == "conditional"
assert any(r.criterion == "avg_latency_ms" for r in verdict2.criteria_results)
def test_judged_pass_rate_fallback_to_pass_rate():
"""If judged_pass_rate is missing, fall back to pass_rate."""
summary = {"pass_rate": 0.96}
criteria = AcceptanceCriteria(judged_pass_rate_min=0.95)
verdict = evaluate_go_no_go(summary, criteria)
assert verdict.decision == "go"
# Should have one result using pass_rate as judged_pass_rate
assert len(verdict.criteria_results) == 1
assert verdict.criteria_results[0].criterion == "judged_pass_rate"
assert verdict.criteria_results[0].actual == 0.96