From 5827c3d3f53e21ed57184500e6fb514209970dc7 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Tue, 25 Aug 2026 14:22:54 +0800 Subject: [PATCH 1/2] =?UTF-8?q?docs(research):=20go/no-go=20=E9=AA=8C?= =?UTF-8?q?=E6=94=B6=E6=8A=A5=E5=91=8A=E6=8A=80=E6=9C=AF=E6=96=B9=E6=A1=88?= =?UTF-8?q?=E8=B0=83=E7=A0=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 调研现有报告生成能力（Run/Campaign/IntelligentEval 三套管线）， 分析 go/no-go 结论的实现路径：纯函数判定核心 + 三级阈值配置 + 渐进式交付。估算总工作量 8 人天，推荐分阶段实施。 Refs: #20 --- research/go-no-go-report.md | 369 ++++++++++++++++++++++++++++++++++++ 1 file changed, 369 insertions(+) create mode 100644 research/go-no-go-report.md diff --git a/research/go-no-go-report.md b/research/go-no-go-report.md new file mode 100644 index 0000000..5f766bf --- /dev/null +++ b/research/go-no-go-report.md @@ -0,0 +1,369 @@ +# Go/No-Go 验收报告技术方案 + +> Ticket: [#20](https://git.solahqb22.cn/solahqb/AgentEvalTool/issues/20) +> Wayfinder: [#13](https://git.solahqb22.cn/solahqb/AgentEvalTool/issues/13) +> 日期: 2026-07-15 + +--- + +## 1. 现有报告能力分析 + +### 1.1 报告体系概览 + +系统当前有三套独立的报告生成管线： + +| 报告类型 | 生成入口 | 渲染输出 | API 端点 | +|---------|---------|---------|---------| +| **单次运行报告** | `evaluation/report.py::generate_report()` | HTML / JSON / Markdown | `GET /api/reports/{run_id}` | +| **活动周期报告** | `evaluation/report.py::generate_campaign_report()` | JSON dict + Markdown | `GET /api/campaigns/{id}/report` | +| **智能评估报告** | `intelligent_eval/report.py::render_report_markdown()` | JSON + Markdown | `GET /api/intelligent-evals/{id}/report` | + +### 1.2 单次运行报告（Run Report） + +**数据源**：`RunSummary` 模型（`models.py`），由 `build_run_summary()` 在引擎执行完毕后一次性写入。 + +**已有指标**： +- `total_cases` / `passed_cases` / `failed_cases` -- 用例级计数 +- `pass_rate` -- 含连通用例的全量通过率 +- `judged_pass_rate` -- 剔除连通用例后的判定型通过率 +- `total_rules` / `passed_rules` -- 规则级计数 +- `avg_latency_ms` -- 平均延迟 +- `case_outcomes` -- 每个用例的 passed/connectivity 权威判定 + +**结论性判断现状**：**无**。报告只呈现原始数字，不做任何阈值比对。HTML 模板只有四个统计卡片（用例总数、通过用例、规则总数、规则通过率），没有"达标/不达标"标记。Markdown 输出同理，只列数据表格。 + +### 1.3 活动周期报告（Campaign Report） + +**已有指标**： +- `overall_pass_rate` / `overall_availability` / `avg_latency_ms` -- 整窗聚合 +- `time_trend` -- 12 时段分桶的通过率/可用性/时延趋势 +- `capability_summary` -- 按场景分组的指标汇总 +- 智能分析（LLM 诊断）：总体结论 + 问题清单 + 改善建议 +- 周期对比：基线 vs 本期的指标 delta + 趋势叙述 + +**结论性判断现状**：**部分**。智能分析的 `overall` 字段包含 LLM 生成的叙述性结论（如"整体表现稳定，建议关注场景 X 的退化"），但这是自然语言判断，不是机械的 go/no-go 判定。没有可配置的阈值比对逻辑。 + +### 1.4 智能评估报告（Intelligent Eval Report） + +**已有指标**： +- `scores.overall` + `scores.dimensions` -- 维度评分 +- `findings` -- 问题发现（含 severity/dimension/evidence） +- `priority_recommendations` -- 改进建议 + +**结论性判断现状**：**无**。有评分但无阈值判定。 + +### 1.5 已有的阈值机制（可复用） + +| 层级 | 阈值 | 位置 | 用途 | +|------|------|------|------| +| 用例级 | `rule_pass_threshold` (默认 0.6) | `Case` 模型 | WEIGHTED 规则逻辑的加权分通过线 | +| 规则级 | `max_ms` | `ResponseTimeRule` | 单轮延迟上限 | +| 规则级 | `keywords_include/exclude` | `KeywordMatchRule` | 关键词匹配 | +| 用例级 | `coherence_min_score` | `Expectation` | 连贯性最低分（隐式规则） | + +**关键发现**：系统已有完善的**用例级**判定链路（`judgement.combine_case_outcome` -> `build_run_summary` -> `case_outcomes`），但**完全缺少运行级/活动级的阈值判定**。go/no-go 需要填补的正是这个空白。 + +--- + +## 2. Go/No-Go 结论的实现方案 + +### 2.1 核心设计 + +新增一个纯函数模块 `evaluation/go_no_go.py`，职责单一：接收报告 dict + 阈值配置 -> 输出结构化结论。 + +```python +# 提议的数据模型 +class AcceptanceCriteria(BaseModel): + """上线验收标准""" + judged_pass_rate_min: float = 0.95 # 判定型通过率下限 + pass_rate_min: float = 0.90 # 全量通过率下限 + avg_latency_max_ms: Optional[float] = None # 平均延迟上限 + availability_min: Optional[float] = None # 可用性下限（活动级） + no_high_severity_finding: bool = False # 不允许有高严重度发现（智能评估） + +class GoNoGoVerdict(BaseModel): + """Go/No-Go 结论""" + decision: str # "go" | "no_go" | "conditional" + summary: str # 人类可读结论，如"通过率 95%，达标，建议上线" + criteria_results: list[CriterionResult] + generated_at: datetime + +class CriterionResult(BaseModel): + """单条标准的比对结果""" + criterion: str # 标准名称 + threshold: float # 阈值 + actual: float # 实际值 + passed: bool # 是否达标 + detail: str # 说明 +``` + +### 2.2 判定逻辑 + +``` +function evaluate_go_no_go(report_dict, criteria): + results = [] + + // 1. 判定型通过率 + if criteria.judged_pass_rate_min is set: + actual = report.summary.judged_pass_rate ?? report.summary.pass_rate + results.append(CriterionResult( + criterion="judged_pass_rate", + threshold=criteria.judged_pass_rate_min, + actual=actual, + passed=actual >= criteria.judged_pass_rate_min + )) + + // 2. 全量通过率 + if criteria.pass_rate_min is set: + actual = report.summary.pass_rate + results.append(...) + + // 3. 平均延迟 + if criteria.avg_latency_max_ms is set: + actual = report.summary.avg_latency_ms + results.append(CriterionResult( + criterion="avg_latency", + threshold=criteria.avg_latency_max_ms, + actual=actual, + passed=actual <= criteria.avg_latency_max_ms + )) + + // 4. 可用性（活动级） + if criteria.availability_min is set: + actual = report.summary.overall_availability + results.append(...) + + // 综合判定 + all_passed = all(r.passed for r in results) + if all_passed: + decision = "go" + summary = f"通过率 {actual*100:.0f}%，达标，建议上线" + elif any critical failures: + decision = "no_go" + summary = f"通过率 {actual*100:.0f}%，未达标（阈值 {threshold*100:.0f}%），不建议上线" + else: + decision = "conditional" + summary = "部分指标达标，存在风险项，建议修复后复测" + + return GoNoGoVerdict(decision, summary, results) +``` + +### 2.3 三级结论语义 + +| 结论 | 条件 | 含义 | +|------|------|------| +| **go** | 全部指标达标 | 建议上线 | +| **no_go** | 核心指标（通过率/可用性）未达标 | 不建议上线 | +| **conditional** | 核心达标但非核心指标（延迟等）有风险 | 建议修复后复测 | + +### 2.4 报告渲染集成 + +**HTML 报告**：在 summary 卡片区域上方新增一个醒目的结论横幅： +- go: 绿色背景 "GO - 建议上线" +- no_go: 红色背景 "NO-GO - 不建议上线" +- conditional: 黄色背景 "CONDITIONAL - 存在风险" + +下方增加一个"验收标准比对表"，逐条列出阈值、实际值、是否达标。 + +**Markdown 报告**：在汇总表格后增加： +```markdown +## 上线验收结论 + +**结论**: GO / NO-GO / CONDITIONAL + +| 指标 | 阈值 | 实际值 | 结果 | +|------|------|--------|------| +| 判定型通过率 | >= 95% | 96.2% | PASS | +| 平均延迟 | <= 5000ms | 3200ms | PASS | +``` + +**JSON 报告**：在顶层新增 `go_no_go` 字段，包含完整的 `GoNoGoVerdict` 结构。 + +### 2.5 API 设计 + +**方案 A（推荐）**：在现有报告端点中自动附带 + +``` +GET /api/reports/{run_id} +-> { ...existing report..., go_no_go: { decision, summary, criteria_results } } +``` + +优点：前端无需改动调用逻辑，结论随报告自动返回。 +缺点：需要知道使用哪套阈值 -> 从 Scenario 或全局配置读取。 + +**方案 B**：独立端点 + +``` +GET /api/reports/{run_id}/verdict?pass_rate_min=0.95&latency_max=5000 +``` + +优点：阈值灵活，可按需传入。 +缺点：增加前端调用复杂度。 + +**推荐方案 A**，因为阈值应该在创建评测时就确定（绑定到 Scenario 或全局配置），而非每次查看报告时指定。 + +--- + +## 3. 达标阈值的配置方式 + +### 3.1 配置层级设计 + +``` +全局默认（Settings） + └── 场景级覆盖（Scenario.acceptance_criteria） + └── 运行级覆盖（EvalRun.acceptance_criteria，可选） +``` + +**优先级**：运行级 > 场景级 > 全局默认 + +### 3.2 具体实现位置 + +| 层级 | 存储位置 | 配置方式 | +|------|---------|---------| +| **全局默认** | `config/settings.py` 新增字段 | 环境变量 `AGENTEVAL_DEFAULT_PASS_RATE_MIN=0.95` | +| **场景级** | `Scenario` 模型新增 `acceptance_criteria: Optional[AcceptanceCriteria]` | API/前端创建场景时配置 | +| **运行级** | `EvalRun` 模型新增 `acceptance_criteria: Optional[AcceptanceCriteria]`（或从 Scenario 继承） | 创建 run 时可选覆盖 | + +### 3.3 全局默认配置示例 + +```python +# settings.py 新增 +default_acceptance_criteria: dict[str, Any] = Field( + default_factory=lambda: { + "judged_pass_rate_min": 0.95, + "pass_rate_min": 0.90, + }, + description="Default acceptance criteria for go/no-go verdicts.", +) +``` + +### 3.4 场景级配置示例 + +在 Scenario 模型中新增可选字段： + +```python +class Scenario(BaseModel): + # ... existing fields ... + acceptance_criteria: Optional[dict[str, Any]] = None + # 例如: {"judged_pass_rate_min": 0.98, "avg_latency_max_ms": 3000} +``` + +前端在场景编辑页增加"验收标准"配置区域（折叠面板），默认展示全局默认值，用户可覆盖。 + +### 3.5 数据库迁移 + +需要 Alembic 迁移脚本为 `scenarios` 表和 `eval_runs` 表添加 `acceptance_criteria` JSON 列。SQLite 的 `render_as_batch=True` 已配置，迁移无特殊障碍。 + +--- + +## 4. 工作量估算 + +### 4.1 后端 + +| 任务 | 人天 | 说明 | +|------|------|------| +| `evaluation/go_no_go.py` 核心判定逻辑 | 0.5 | 纯函数，单测覆盖 | +| `AcceptanceCriteria` / `GoNoGoVerdict` 数据模型 | 0.5 | Pydantic 模型 | +| `RunSummary` / `Scenario` 模型扩展 | 0.5 | 新增字段 + 兼容处理 | +| `generate_report()` 集成 go_no_go 结论 | 0.5 | 在报告 dict 中附加 verdict | +| `report_render.py` 三种格式渲染 | 1.0 | HTML 横幅 + Markdown 表格 + JSON 字段 | +| `settings.py` 全局默认配置 | 0.25 | 新增 Settings 字段 | +| Alembic 迁移 | 0.25 | scenarios + eval_runs 加列 | +| API 端点调整 | 0.5 | reports router 附带 verdict | +| 单元测试 + 集成测试 | 1.0 | go_no_go 逻辑 + API 覆盖 | +| **后端小计** | **5.0** | | + +### 4.2 前端 + +| 任务 | 人天 | 说明 | +|------|------|------| +| 场景编辑页增加"验收标准"配置 | 1.0 | FormDrawer 内新增折叠面板 | +| Run 报告详情页展示 go/no-go 结论 | 1.0 | 结论横幅 + 比对表格 | +| 活动报告页展示活动级结论 | 0.5 | 复用组件 | +| **前端小计** | **2.5** | | + +### 4.3 总计 + +| 模块 | 人天 | +|------|------| +| 后端 | 5.0 | +| 前端 | 2.5 | +| 联调 + 部署 | 0.5 | +| **总计** | **8.0 人天** | + +如果只做后端（API 返回 verdict，前端后续迭代），可压缩到 **5.0 人天**。 + +--- + +## 5. 推荐方案 + +### 5.1 推荐：方案 A + 三级配置 + 渐进式交付 + +**理由**： + +1. **纯函数核心**（`go_no_go.py`）：与现有架构一致（`metrics.py` / `run_summary.py` 都是纯函数），易测试、易扩展。 + +2. **三级阈值配置**（全局默认 -> 场景级 -> 运行级）： + - 全局默认保证开箱即用，不需要每个场景都配置 + - 场景级覆盖满足差异化需求（如关键场景要求 98% 通过率） + - 运行级覆盖保留灵活性（如临时加严测试） + +3. **自动附带而非独立端点**：前端零改动即可获得 verdict 数据，降低集成成本。 + +4. **三级结论（go/no_go/conditional）**：比二元判定更实用。"conditional" 覆盖了"通过率达标但延迟偏高"这类常见场景，避免误判。 + +5. **复用现有指标**：不需要新增数据采集，`judged_pass_rate` / `pass_rate` / `avg_latency_ms` / `availability` 均已由引擎计算并持久化。go/no-go 只是在读路径上增加一层阈值比对。 + +### 5.2 实施路径 + +``` +Phase 1（MVP，5 人天）: + - go_no_go.py 核心逻辑 + - 全局默认阈值（Settings） + - generate_report() 集成 + - JSON/Markdown 渲染 + - 单元测试 + +Phase 2（完善，3 人天）: + - 场景级 acceptance_criteria 配置 + - HTML 渲染（结论横幅） + - 前端场景编辑页配置面板 + - 前端报告展示页结论展示 + +Phase 3（可选增强）: + - 活动级 go/no-go（跨 run 聚合判定） + - 智能评估报告的评分阈值集成 + - Webhook 推送 verdict（CI/CD 集成） +``` + +### 5.3 关键决策点 + +| 决策 | 推荐 | 备选 | 理由 | +|------|------|------|------| +| 核心指标选择 | `judged_pass_rate` 为主 | `pass_rate` | 连通用例无判定意义，`judged_pass_rate` 更准确反映质量 | +| 阈值存储 | JSON 列（灵活） | 独立表（规范化） | 阈值结构简单且固定，JSON 足够，避免过度设计 | +| 结论渲染位置 | 报告顶部横幅 | 报告底部 | 结论应第一时间可见，类似体检报告的"总结" | +| 活动级 go/no-go | Phase 3 再做 | 同期实现 | 活动级需要跨 run 聚合，复杂度较高，且 ticket #20 聚焦单次验收 | + +### 5.4 与现有架构的契合度 + +- **ADR-0002（通过率口径）**：go/no-go 直接消费 `judged_pass_rate`，口径一致 +- **ADR-0004（聚合口径）**：活动级聚合复用 `aggregate_runs`，不重算 +- **规则注册表模式**：go_no_go 判定器可设计为可扩展的（未来可能增加新指标） +- **纯函数渲染**：遵循 `report_render.py` 的 dict-in/string-out 模式 + +--- + +## 附录：关键代码路径 + +| 文件 | 职责 | +|------|------| +| `backend/agenteval/evaluation/report.py` | 报告生成（`generate_report` / `generate_campaign_report`） | +| `backend/agenteval/evaluation/report_render.py` | 报告渲染（HTML/Markdown/JSON） | +| `backend/agenteval/evaluation/run_summary.py` | 运行汇总（`build_run_summary`） | +| `backend/agenteval/evaluation/metrics.py` | 跨运行聚合（`aggregate_runs`） | +| `backend/agenteval/evaluation/judgement.py` | 用例判定（`combine_case_outcome`） | +| `backend/agenteval/evaluation/case_verdict.py` | 用例判定读路径（`resolve_case_verdicts`） | +| `backend/agenteval/models.py` | 数据模型（`RunSummary` / `Scenario` / `Case`） | +| `backend/agenteval/config/settings.py` | 全局配置 | +| `backend/agenteval/web/routers/reports.py` | 报告 API 端点 | -- 2.45.2 From 2314ebe3bc38559cafc3601d7e8084475371005a Mon Sep 17 00:00:00 2001 From: sinohqb Date: Tue, 25 Aug 2026 14:28:55 +0800 Subject: [PATCH 2/2] =?UTF-8?q?feat(report):=20=E4=B8=8A=E7=BA=BF=E9=AA=8C?= =?UTF-8?q?=E6=94=B6=E6=8A=A5=E5=91=8A=20go/no-go=20=E7=BB=93=E8=AE=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增纯函数模块 evaluation/go_no_go.py，基于 RunSummary 指标自动生成 go/no-go/conditional 三级结论。 - AcceptanceCriteria 模型：judged_pass_rate_min、pass_rate_min、avg_latency_max_ms - GoNoGoVerdict 模型：decision、summary、criteria_results - evaluate_go_no_go() 纯函数：接收 summary dict + criteria -> 返回 verdict - 集成到 generate_report()，报告 dict 自动附带 go_no_go 字段 - 三级结论：go（全部达标）、no_go（核心未达标）、conditional（有风险） - 新增 9 项单元测试（642 tests passed） Closes #20 --- backend/agenteval/evaluation/go_no_go.py | 137 ++++++++++++++++++++++ backend/agenteval/evaluation/report.py | 28 +++-- tests/unit/test_go_no_go.py | 138 +++++++++++++++++++++++ 3 files changed, 293 insertions(+), 10 deletions(-) create mode 100644 backend/agenteval/evaluation/go_no_go.py create mode 100644 tests/unit/test_go_no_go.py diff --git a/backend/agenteval/evaluation/go_no_go.py b/backend/agenteval/evaluation/go_no_go.py new file mode 100644 index 0000000..b0afb2f --- /dev/null +++ b/backend/agenteval/evaluation/go_no_go.py @@ -0,0 +1,137 @@ +"""Go/No-Go acceptance verdict for evaluation runs.""" + +from datetime import datetime, timezone +from typing import Any, Optional + +from pydantic import BaseModel, Field + + +class AcceptanceCriteria(BaseModel): + """Acceptance criteria for go/no-go verdict.""" + + judged_pass_rate_min: float = Field(default=0.95, description="Minimum judged pass rate (0-1)") + pass_rate_min: float = Field(default=0.90, description="Minimum overall pass rate (0-1)") + avg_latency_max_ms: Optional[float] = Field(default=None, description="Maximum average latency in ms") + availability_min: Optional[float] = Field(default=None, description="Minimum availability (0-1), for campaign level") + + +class CriterionResult(BaseModel): + """Result of checking a single criterion.""" + + criterion: str = Field(description="Criterion name") + threshold: float = Field(description="Threshold value") + actual: float = Field(description="Actual value") + passed: bool = Field(description="Whether the criterion passed") + detail: str = Field(default="", description="Human-readable detail") + + +class GoNoGoVerdict(BaseModel): + """Go/No-Go verdict for an evaluation run.""" + + decision: str = Field(description="go | no_go | conditional") + summary: str = Field(description="Human-readable summary") + criteria_results: list[CriterionResult] = Field(default_factory=list) + generated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +def evaluate_go_no_go( + summary: dict[str, Any], + criteria: AcceptanceCriteria | None = None, +) -> GoNoGoVerdict: + """Evaluate go/no-go verdict based on run summary and acceptance criteria. + + Args: + summary: Run summary dict with keys like judged_pass_rate, pass_rate, avg_latency_ms + criteria: Acceptance criteria. If None, uses defaults. + + Returns: + GoNoGoVerdict with decision, summary, and criteria results. + """ + if criteria is None: + criteria = AcceptanceCriteria() + + results: list[CriterionResult] = [] + + # Check judged pass rate (only if present in summary) + judged_pass_rate = summary.get("judged_pass_rate") + if judged_pass_rate is None: + judged_pass_rate = summary.get("pass_rate") + if judged_pass_rate is not None: + passed = judged_pass_rate >= criteria.judged_pass_rate_min + results.append(CriterionResult( + criterion="judged_pass_rate", + threshold=criteria.judged_pass_rate_min, + actual=judged_pass_rate, + passed=passed, + detail=f"判定型通过率 {judged_pass_rate*100:.1f}% {'>=' if passed else '<'} {criteria.judged_pass_rate_min*100:.0f}%", + )) + + # Check overall pass rate (only if different from judged and present) + pass_rate = summary.get("pass_rate") + if pass_rate is not None and pass_rate != judged_pass_rate: + passed = pass_rate >= criteria.pass_rate_min + results.append(CriterionResult( + criterion="pass_rate", + threshold=criteria.pass_rate_min, + actual=pass_rate, + passed=passed, + detail=f"全量通过率 {pass_rate*100:.1f}% {'>=' if passed else '<'} {criteria.pass_rate_min*100:.0f}%", + )) + + # Check average latency + avg_latency_ms = summary.get("avg_latency_ms") + if avg_latency_ms is not None and criteria.avg_latency_max_ms is not None: + passed = avg_latency_ms <= criteria.avg_latency_max_ms + results.append(CriterionResult( + criterion="avg_latency_ms", + threshold=criteria.avg_latency_max_ms, + actual=avg_latency_ms, + passed=passed, + detail=f"平均延迟 {avg_latency_ms:.0f}ms {'<=' if passed else '>'} {criteria.avg_latency_max_ms:.0f}ms", + )) + + # Check availability (for campaign level) + availability = summary.get("overall_availability") or summary.get("availability") + if availability is not None and criteria.availability_min is not None: + passed = availability >= criteria.availability_min + results.append(CriterionResult( + criterion="availability", + threshold=criteria.availability_min, + actual=availability, + passed=passed, + detail=f"可用性 {availability*100:.1f}% {'>=' if passed else '<'} {criteria.availability_min*100:.0f}%", + )) + + # Determine overall decision + if not results: + return GoNoGoVerdict( + decision="conditional", + summary="无可用指标进行评估", + criteria_results=[], + ) + + all_passed = all(r.passed for r in results) + if all_passed: + decision = "go" + main_metric = results[0] + summary_text = f"通过率 {main_metric.actual*100:.0f}%，达标，建议上线" + else: + # Check if core metrics (pass rate) failed + core_failed = any( + not r.passed and r.criterion in ("judged_pass_rate", "pass_rate", "availability") + for r in results + ) + if core_failed: + decision = "no_go" + failed_metrics = [r for r in results if not r.passed] + summary_text = f"核心指标未达标（{', '.join(r.criterion for r in failed_metrics)}），不建议上线" + else: + decision = "conditional" + risky_metrics = [r for r in results if not r.passed] + summary_text = f"部分指标达标，存在风险项（{', '.join(r.criterion for r in risky_metrics)}），建议修复后复测" + + return GoNoGoVerdict( + decision=decision, + summary=summary_text, + criteria_results=results, + ) diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index 255e7a7..8e3ba82 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -12,6 +12,7 @@ from typing import Any, Optional from sqlmodel import Session from agenteval.evaluation.case_verdict import build_case_evidence, resolve_case_verdicts +from agenteval.evaluation.go_no_go import evaluate_go_no_go from agenteval.evaluation.metrics import aggregate_runs from agenteval.evaluation.report_render import render_html, render_json, render_markdown from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary @@ -103,6 +104,21 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: if judged_pass_rate is None and judged_total > 0: judged_pass_rate = round((passed_cases - connectivity_count) / judged_total, 4) + summary_dict = { + "total_cases": total_cases, + "passed_cases": passed_cases, + "failed_cases": summary.failed_cases, + "total_rules": summary.total_rules, + "passed_rules": summary.passed_rules, + "pass_rate": summary.pass_rate if summary.pass_rate is not None else 0.0, + "connectivity_cases": connectivity_count, + "judged_pass_rate": judged_pass_rate, + "avg_latency_ms": summary.avg_latency_ms, + } + + # Generate go/no-go verdict + verdict = evaluate_go_no_go(summary_dict) + return { "run_id": run.id, "target_id": run.target_id, @@ -114,16 +130,8 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: "status": run.status.value, "started_at": iso_utc(run.started_at), "completed_at": iso_utc(run.completed_at), - "summary": { - "total_cases": total_cases, - "passed_cases": passed_cases, - "failed_cases": summary.failed_cases, - "total_rules": summary.total_rules, - "passed_rules": summary.passed_rules, - "pass_rate": summary.pass_rate if summary.pass_rate is not None else 0.0, - "connectivity_cases": connectivity_count, - "judged_pass_rate": judged_pass_rate, - }, + "summary": summary_dict, + "go_no_go": verdict.model_dump(mode="json"), "cases": cases, } diff --git a/tests/unit/test_go_no_go.py b/tests/unit/test_go_no_go.py new file mode 100644 index 0000000..fd10f67 --- /dev/null +++ b/tests/unit/test_go_no_go.py @@ -0,0 +1,138 @@ +"""Tests for go/no-go acceptance verdict.""" + +import pytest + +from agenteval.evaluation.go_no_go import ( + AcceptanceCriteria, + CriterionResult, + GoNoGoVerdict, + evaluate_go_no_go, +) + + +def test_go_verdict_all_pass(): + """All criteria met -> go.""" + summary = { + "judged_pass_rate": 0.96, + "pass_rate": 0.95, + "avg_latency_ms": 2000, + } + criteria = AcceptanceCriteria( + judged_pass_rate_min=0.95, + pass_rate_min=0.90, + avg_latency_max_ms=5000, + ) + verdict = evaluate_go_no_go(summary, criteria) + assert verdict.decision == "go" + assert "达标" in verdict.summary + assert all(r.passed for r in verdict.criteria_results) + + +def test_no_go_verdict_core_failed(): + """Core metric (pass rate) failed -> no_go.""" + summary = { + "judged_pass_rate": 0.80, + "pass_rate": 0.75, + } + criteria = AcceptanceCriteria( + judged_pass_rate_min=0.95, + pass_rate_min=0.90, + ) + verdict = evaluate_go_no_go(summary, criteria) + assert verdict.decision == "no_go" + assert "不建议上线" in verdict.summary + assert any(not r.passed for r in verdict.criteria_results) + + +def test_conditional_verdict_non_core_risk(): + """Core passed but non-core (latency) failed -> conditional.""" + summary = { + "judged_pass_rate": 0.96, + "pass_rate": 0.95, + "avg_latency_ms": 8000, + } + criteria = AcceptanceCriteria( + judged_pass_rate_min=0.95, + pass_rate_min=0.90, + avg_latency_max_ms=5000, + ) + verdict = evaluate_go_no_go(summary, criteria) + assert verdict.decision == "conditional" + assert "风险" in verdict.summary + + +def test_default_criteria(): + """Default criteria should be applied when None.""" + summary = { + "judged_pass_rate": 0.96, + "pass_rate": 0.95, + } + verdict = evaluate_go_no_go(summary, None) + assert verdict.decision == "go" + assert len(verdict.criteria_results) >= 1 + + +def test_empty_summary(): + """Empty summary -> conditional with no results.""" + verdict = evaluate_go_no_go({}, None) + assert verdict.decision == "conditional" + assert "无可用指标" in verdict.summary + assert len(verdict.criteria_results) == 0 + + +def test_criterion_result_model(): + """CriterionResult model should work correctly.""" + result = CriterionResult( + criterion="pass_rate", + threshold=0.95, + actual=0.96, + passed=True, + detail="通过率 96% >= 95%", + ) + assert result.criterion == "pass_rate" + assert result.passed is True + + +def test_verdict_model(): + """GoNoGoVerdict model should work correctly.""" + verdict = GoNoGoVerdict( + decision="go", + summary="测试通过", + criteria_results=[], + ) + assert verdict.decision == "go" + assert verdict.generated_at is not None + + +def test_latency_only_when_configured(): + """Latency should only be checked when avg_latency_max_ms is set.""" + summary = { + "judged_pass_rate": 0.96, + "avg_latency_ms": 10000, # High latency + } + # Without latency threshold + criteria_no_latency = AcceptanceCriteria(judged_pass_rate_min=0.95) + verdict1 = evaluate_go_no_go(summary, criteria_no_latency) + assert verdict1.decision == "go" + assert all(r.criterion != "avg_latency_ms" for r in verdict1.criteria_results) + + # With latency threshold + criteria_with_latency = AcceptanceCriteria( + judged_pass_rate_min=0.95, + avg_latency_max_ms=5000, + ) + verdict2 = evaluate_go_no_go(summary, criteria_with_latency) + assert verdict2.decision == "conditional" + assert any(r.criterion == "avg_latency_ms" for r in verdict2.criteria_results) + + +def test_judged_pass_rate_fallback_to_pass_rate(): + """If judged_pass_rate is missing, fall back to pass_rate.""" + summary = {"pass_rate": 0.96} + criteria = AcceptanceCriteria(judged_pass_rate_min=0.95) + verdict = evaluate_go_no_go(summary, criteria) + assert verdict.decision == "go" + # Should have one result using pass_rate as judged_pass_rate + assert len(verdict.criteria_results) == 1 + assert verdict.criteria_results[0].criterion == "judged_pass_rate" + assert verdict.criteria_results[0].actual == 0.96 -- 2.45.2