Compare commits
3 Commits
a894ed7179
...
2f08e7bf06
| Author | SHA1 | Date | |
|---|---|---|---|
| 2f08e7bf06 | |||
|
|
2314ebe3bc | ||
|
|
5827c3d3f5 |
137
backend/agenteval/evaluation/go_no_go.py
Normal file
137
backend/agenteval/evaluation/go_no_go.py
Normal file
@ -0,0 +1,137 @@
|
|||||||
|
"""Go/No-Go acceptance verdict for evaluation runs."""
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
|
||||||
|
class AcceptanceCriteria(BaseModel):
|
||||||
|
"""Acceptance criteria for go/no-go verdict."""
|
||||||
|
|
||||||
|
judged_pass_rate_min: float = Field(default=0.95, description="Minimum judged pass rate (0-1)")
|
||||||
|
pass_rate_min: float = Field(default=0.90, description="Minimum overall pass rate (0-1)")
|
||||||
|
avg_latency_max_ms: Optional[float] = Field(default=None, description="Maximum average latency in ms")
|
||||||
|
availability_min: Optional[float] = Field(default=None, description="Minimum availability (0-1), for campaign level")
|
||||||
|
|
||||||
|
|
||||||
|
class CriterionResult(BaseModel):
|
||||||
|
"""Result of checking a single criterion."""
|
||||||
|
|
||||||
|
criterion: str = Field(description="Criterion name")
|
||||||
|
threshold: float = Field(description="Threshold value")
|
||||||
|
actual: float = Field(description="Actual value")
|
||||||
|
passed: bool = Field(description="Whether the criterion passed")
|
||||||
|
detail: str = Field(default="", description="Human-readable detail")
|
||||||
|
|
||||||
|
|
||||||
|
class GoNoGoVerdict(BaseModel):
|
||||||
|
"""Go/No-Go verdict for an evaluation run."""
|
||||||
|
|
||||||
|
decision: str = Field(description="go | no_go | conditional")
|
||||||
|
summary: str = Field(description="Human-readable summary")
|
||||||
|
criteria_results: list[CriterionResult] = Field(default_factory=list)
|
||||||
|
generated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_go_no_go(
|
||||||
|
summary: dict[str, Any],
|
||||||
|
criteria: AcceptanceCriteria | None = None,
|
||||||
|
) -> GoNoGoVerdict:
|
||||||
|
"""Evaluate go/no-go verdict based on run summary and acceptance criteria.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
summary: Run summary dict with keys like judged_pass_rate, pass_rate, avg_latency_ms
|
||||||
|
criteria: Acceptance criteria. If None, uses defaults.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
GoNoGoVerdict with decision, summary, and criteria results.
|
||||||
|
"""
|
||||||
|
if criteria is None:
|
||||||
|
criteria = AcceptanceCriteria()
|
||||||
|
|
||||||
|
results: list[CriterionResult] = []
|
||||||
|
|
||||||
|
# Check judged pass rate (only if present in summary)
|
||||||
|
judged_pass_rate = summary.get("judged_pass_rate")
|
||||||
|
if judged_pass_rate is None:
|
||||||
|
judged_pass_rate = summary.get("pass_rate")
|
||||||
|
if judged_pass_rate is not None:
|
||||||
|
passed = judged_pass_rate >= criteria.judged_pass_rate_min
|
||||||
|
results.append(CriterionResult(
|
||||||
|
criterion="judged_pass_rate",
|
||||||
|
threshold=criteria.judged_pass_rate_min,
|
||||||
|
actual=judged_pass_rate,
|
||||||
|
passed=passed,
|
||||||
|
detail=f"判定型通过率 {judged_pass_rate*100:.1f}% {'>=' if passed else '<'} {criteria.judged_pass_rate_min*100:.0f}%",
|
||||||
|
))
|
||||||
|
|
||||||
|
# Check overall pass rate (only if different from judged and present)
|
||||||
|
pass_rate = summary.get("pass_rate")
|
||||||
|
if pass_rate is not None and pass_rate != judged_pass_rate:
|
||||||
|
passed = pass_rate >= criteria.pass_rate_min
|
||||||
|
results.append(CriterionResult(
|
||||||
|
criterion="pass_rate",
|
||||||
|
threshold=criteria.pass_rate_min,
|
||||||
|
actual=pass_rate,
|
||||||
|
passed=passed,
|
||||||
|
detail=f"全量通过率 {pass_rate*100:.1f}% {'>=' if passed else '<'} {criteria.pass_rate_min*100:.0f}%",
|
||||||
|
))
|
||||||
|
|
||||||
|
# Check average latency
|
||||||
|
avg_latency_ms = summary.get("avg_latency_ms")
|
||||||
|
if avg_latency_ms is not None and criteria.avg_latency_max_ms is not None:
|
||||||
|
passed = avg_latency_ms <= criteria.avg_latency_max_ms
|
||||||
|
results.append(CriterionResult(
|
||||||
|
criterion="avg_latency_ms",
|
||||||
|
threshold=criteria.avg_latency_max_ms,
|
||||||
|
actual=avg_latency_ms,
|
||||||
|
passed=passed,
|
||||||
|
detail=f"平均延迟 {avg_latency_ms:.0f}ms {'<=' if passed else '>'} {criteria.avg_latency_max_ms:.0f}ms",
|
||||||
|
))
|
||||||
|
|
||||||
|
# Check availability (for campaign level)
|
||||||
|
availability = summary.get("overall_availability") or summary.get("availability")
|
||||||
|
if availability is not None and criteria.availability_min is not None:
|
||||||
|
passed = availability >= criteria.availability_min
|
||||||
|
results.append(CriterionResult(
|
||||||
|
criterion="availability",
|
||||||
|
threshold=criteria.availability_min,
|
||||||
|
actual=availability,
|
||||||
|
passed=passed,
|
||||||
|
detail=f"可用性 {availability*100:.1f}% {'>=' if passed else '<'} {criteria.availability_min*100:.0f}%",
|
||||||
|
))
|
||||||
|
|
||||||
|
# Determine overall decision
|
||||||
|
if not results:
|
||||||
|
return GoNoGoVerdict(
|
||||||
|
decision="conditional",
|
||||||
|
summary="无可用指标进行评估",
|
||||||
|
criteria_results=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
all_passed = all(r.passed for r in results)
|
||||||
|
if all_passed:
|
||||||
|
decision = "go"
|
||||||
|
main_metric = results[0]
|
||||||
|
summary_text = f"通过率 {main_metric.actual*100:.0f}%,达标,建议上线"
|
||||||
|
else:
|
||||||
|
# Check if core metrics (pass rate) failed
|
||||||
|
core_failed = any(
|
||||||
|
not r.passed and r.criterion in ("judged_pass_rate", "pass_rate", "availability")
|
||||||
|
for r in results
|
||||||
|
)
|
||||||
|
if core_failed:
|
||||||
|
decision = "no_go"
|
||||||
|
failed_metrics = [r for r in results if not r.passed]
|
||||||
|
summary_text = f"核心指标未达标({', '.join(r.criterion for r in failed_metrics)}),不建议上线"
|
||||||
|
else:
|
||||||
|
decision = "conditional"
|
||||||
|
risky_metrics = [r for r in results if not r.passed]
|
||||||
|
summary_text = f"部分指标达标,存在风险项({', '.join(r.criterion for r in risky_metrics)}),建议修复后复测"
|
||||||
|
|
||||||
|
return GoNoGoVerdict(
|
||||||
|
decision=decision,
|
||||||
|
summary=summary_text,
|
||||||
|
criteria_results=results,
|
||||||
|
)
|
||||||
@ -12,6 +12,7 @@ from typing import Any, Optional
|
|||||||
from sqlmodel import Session
|
from sqlmodel import Session
|
||||||
|
|
||||||
from agenteval.evaluation.case_verdict import build_case_evidence, resolve_case_verdicts
|
from agenteval.evaluation.case_verdict import build_case_evidence, resolve_case_verdicts
|
||||||
|
from agenteval.evaluation.go_no_go import evaluate_go_no_go
|
||||||
from agenteval.evaluation.metrics import aggregate_runs
|
from agenteval.evaluation.metrics import aggregate_runs
|
||||||
from agenteval.evaluation.report_render import render_html, render_json, render_markdown
|
from agenteval.evaluation.report_render import render_html, render_json, render_markdown
|
||||||
from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary
|
from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary
|
||||||
@ -103,6 +104,21 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
|
|||||||
if judged_pass_rate is None and judged_total > 0:
|
if judged_pass_rate is None and judged_total > 0:
|
||||||
judged_pass_rate = round((passed_cases - connectivity_count) / judged_total, 4)
|
judged_pass_rate = round((passed_cases - connectivity_count) / judged_total, 4)
|
||||||
|
|
||||||
|
summary_dict = {
|
||||||
|
"total_cases": total_cases,
|
||||||
|
"passed_cases": passed_cases,
|
||||||
|
"failed_cases": summary.failed_cases,
|
||||||
|
"total_rules": summary.total_rules,
|
||||||
|
"passed_rules": summary.passed_rules,
|
||||||
|
"pass_rate": summary.pass_rate if summary.pass_rate is not None else 0.0,
|
||||||
|
"connectivity_cases": connectivity_count,
|
||||||
|
"judged_pass_rate": judged_pass_rate,
|
||||||
|
"avg_latency_ms": summary.avg_latency_ms,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Generate go/no-go verdict
|
||||||
|
verdict = evaluate_go_no_go(summary_dict)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"run_id": run.id,
|
"run_id": run.id,
|
||||||
"target_id": run.target_id,
|
"target_id": run.target_id,
|
||||||
@ -114,16 +130,8 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
|
|||||||
"status": run.status.value,
|
"status": run.status.value,
|
||||||
"started_at": iso_utc(run.started_at),
|
"started_at": iso_utc(run.started_at),
|
||||||
"completed_at": iso_utc(run.completed_at),
|
"completed_at": iso_utc(run.completed_at),
|
||||||
"summary": {
|
"summary": summary_dict,
|
||||||
"total_cases": total_cases,
|
"go_no_go": verdict.model_dump(mode="json"),
|
||||||
"passed_cases": passed_cases,
|
|
||||||
"failed_cases": summary.failed_cases,
|
|
||||||
"total_rules": summary.total_rules,
|
|
||||||
"passed_rules": summary.passed_rules,
|
|
||||||
"pass_rate": summary.pass_rate if summary.pass_rate is not None else 0.0,
|
|
||||||
"connectivity_cases": connectivity_count,
|
|
||||||
"judged_pass_rate": judged_pass_rate,
|
|
||||||
},
|
|
||||||
"cases": cases,
|
"cases": cases,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
369
research/go-no-go-report.md
Normal file
369
research/go-no-go-report.md
Normal file
@ -0,0 +1,369 @@
|
|||||||
|
# Go/No-Go 验收报告技术方案
|
||||||
|
|
||||||
|
> Ticket: [#20](https://git.solahqb22.cn/solahqb/AgentEvalTool/issues/20)
|
||||||
|
> Wayfinder: [#13](https://git.solahqb22.cn/solahqb/AgentEvalTool/issues/13)
|
||||||
|
> 日期: 2026-07-15
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 现有报告能力分析
|
||||||
|
|
||||||
|
### 1.1 报告体系概览
|
||||||
|
|
||||||
|
系统当前有三套独立的报告生成管线:
|
||||||
|
|
||||||
|
| 报告类型 | 生成入口 | 渲染输出 | API 端点 |
|
||||||
|
|---------|---------|---------|---------|
|
||||||
|
| **单次运行报告** | `evaluation/report.py::generate_report()` | HTML / JSON / Markdown | `GET /api/reports/{run_id}` |
|
||||||
|
| **活动周期报告** | `evaluation/report.py::generate_campaign_report()` | JSON dict + Markdown | `GET /api/campaigns/{id}/report` |
|
||||||
|
| **智能评估报告** | `intelligent_eval/report.py::render_report_markdown()` | JSON + Markdown | `GET /api/intelligent-evals/{id}/report` |
|
||||||
|
|
||||||
|
### 1.2 单次运行报告(Run Report)
|
||||||
|
|
||||||
|
**数据源**:`RunSummary` 模型(`models.py`),由 `build_run_summary()` 在引擎执行完毕后一次性写入。
|
||||||
|
|
||||||
|
**已有指标**:
|
||||||
|
- `total_cases` / `passed_cases` / `failed_cases` -- 用例级计数
|
||||||
|
- `pass_rate` -- 含连通用例的全量通过率
|
||||||
|
- `judged_pass_rate` -- 剔除连通用例后的判定型通过率
|
||||||
|
- `total_rules` / `passed_rules` -- 规则级计数
|
||||||
|
- `avg_latency_ms` -- 平均延迟
|
||||||
|
- `case_outcomes` -- 每个用例的 passed/connectivity 权威判定
|
||||||
|
|
||||||
|
**结论性判断现状**:**无**。报告只呈现原始数字,不做任何阈值比对。HTML 模板只有四个统计卡片(用例总数、通过用例、规则总数、规则通过率),没有"达标/不达标"标记。Markdown 输出同理,只列数据表格。
|
||||||
|
|
||||||
|
### 1.3 活动周期报告(Campaign Report)
|
||||||
|
|
||||||
|
**已有指标**:
|
||||||
|
- `overall_pass_rate` / `overall_availability` / `avg_latency_ms` -- 整窗聚合
|
||||||
|
- `time_trend` -- 12 时段分桶的通过率/可用性/时延趋势
|
||||||
|
- `capability_summary` -- 按场景分组的指标汇总
|
||||||
|
- 智能分析(LLM 诊断):总体结论 + 问题清单 + 改善建议
|
||||||
|
- 周期对比:基线 vs 本期的指标 delta + 趋势叙述
|
||||||
|
|
||||||
|
**结论性判断现状**:**部分**。智能分析的 `overall` 字段包含 LLM 生成的叙述性结论(如"整体表现稳定,建议关注场景 X 的退化"),但这是自然语言判断,不是机械的 go/no-go 判定。没有可配置的阈值比对逻辑。
|
||||||
|
|
||||||
|
### 1.4 智能评估报告(Intelligent Eval Report)
|
||||||
|
|
||||||
|
**已有指标**:
|
||||||
|
- `scores.overall` + `scores.dimensions` -- 维度评分
|
||||||
|
- `findings` -- 问题发现(含 severity/dimension/evidence)
|
||||||
|
- `priority_recommendations` -- 改进建议
|
||||||
|
|
||||||
|
**结论性判断现状**:**无**。有评分但无阈值判定。
|
||||||
|
|
||||||
|
### 1.5 已有的阈值机制(可复用)
|
||||||
|
|
||||||
|
| 层级 | 阈值 | 位置 | 用途 |
|
||||||
|
|------|------|------|------|
|
||||||
|
| 用例级 | `rule_pass_threshold` (默认 0.6) | `Case` 模型 | WEIGHTED 规则逻辑的加权分通过线 |
|
||||||
|
| 规则级 | `max_ms` | `ResponseTimeRule` | 单轮延迟上限 |
|
||||||
|
| 规则级 | `keywords_include/exclude` | `KeywordMatchRule` | 关键词匹配 |
|
||||||
|
| 用例级 | `coherence_min_score` | `Expectation` | 连贯性最低分(隐式规则) |
|
||||||
|
|
||||||
|
**关键发现**:系统已有完善的**用例级**判定链路(`judgement.combine_case_outcome` -> `build_run_summary` -> `case_outcomes`),但**完全缺少运行级/活动级的阈值判定**。go/no-go 需要填补的正是这个空白。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Go/No-Go 结论的实现方案
|
||||||
|
|
||||||
|
### 2.1 核心设计
|
||||||
|
|
||||||
|
新增一个纯函数模块 `evaluation/go_no_go.py`,职责单一:接收报告 dict + 阈值配置 -> 输出结构化结论。
|
||||||
|
|
||||||
|
```python
|
||||||
|
# 提议的数据模型
|
||||||
|
class AcceptanceCriteria(BaseModel):
|
||||||
|
"""上线验收标准"""
|
||||||
|
judged_pass_rate_min: float = 0.95 # 判定型通过率下限
|
||||||
|
pass_rate_min: float = 0.90 # 全量通过率下限
|
||||||
|
avg_latency_max_ms: Optional[float] = None # 平均延迟上限
|
||||||
|
availability_min: Optional[float] = None # 可用性下限(活动级)
|
||||||
|
no_high_severity_finding: bool = False # 不允许有高严重度发现(智能评估)
|
||||||
|
|
||||||
|
class GoNoGoVerdict(BaseModel):
|
||||||
|
"""Go/No-Go 结论"""
|
||||||
|
decision: str # "go" | "no_go" | "conditional"
|
||||||
|
summary: str # 人类可读结论,如"通过率 95%,达标,建议上线"
|
||||||
|
criteria_results: list[CriterionResult]
|
||||||
|
generated_at: datetime
|
||||||
|
|
||||||
|
class CriterionResult(BaseModel):
|
||||||
|
"""单条标准的比对结果"""
|
||||||
|
criterion: str # 标准名称
|
||||||
|
threshold: float # 阈值
|
||||||
|
actual: float # 实际值
|
||||||
|
passed: bool # 是否达标
|
||||||
|
detail: str # 说明
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 判定逻辑
|
||||||
|
|
||||||
|
```
|
||||||
|
function evaluate_go_no_go(report_dict, criteria):
|
||||||
|
results = []
|
||||||
|
|
||||||
|
// 1. 判定型通过率
|
||||||
|
if criteria.judged_pass_rate_min is set:
|
||||||
|
actual = report.summary.judged_pass_rate ?? report.summary.pass_rate
|
||||||
|
results.append(CriterionResult(
|
||||||
|
criterion="judged_pass_rate",
|
||||||
|
threshold=criteria.judged_pass_rate_min,
|
||||||
|
actual=actual,
|
||||||
|
passed=actual >= criteria.judged_pass_rate_min
|
||||||
|
))
|
||||||
|
|
||||||
|
// 2. 全量通过率
|
||||||
|
if criteria.pass_rate_min is set:
|
||||||
|
actual = report.summary.pass_rate
|
||||||
|
results.append(...)
|
||||||
|
|
||||||
|
// 3. 平均延迟
|
||||||
|
if criteria.avg_latency_max_ms is set:
|
||||||
|
actual = report.summary.avg_latency_ms
|
||||||
|
results.append(CriterionResult(
|
||||||
|
criterion="avg_latency",
|
||||||
|
threshold=criteria.avg_latency_max_ms,
|
||||||
|
actual=actual,
|
||||||
|
passed=actual <= criteria.avg_latency_max_ms
|
||||||
|
))
|
||||||
|
|
||||||
|
// 4. 可用性(活动级)
|
||||||
|
if criteria.availability_min is set:
|
||||||
|
actual = report.summary.overall_availability
|
||||||
|
results.append(...)
|
||||||
|
|
||||||
|
// 综合判定
|
||||||
|
all_passed = all(r.passed for r in results)
|
||||||
|
if all_passed:
|
||||||
|
decision = "go"
|
||||||
|
summary = f"通过率 {actual*100:.0f}%,达标,建议上线"
|
||||||
|
elif any critical failures:
|
||||||
|
decision = "no_go"
|
||||||
|
summary = f"通过率 {actual*100:.0f}%,未达标(阈值 {threshold*100:.0f}%),不建议上线"
|
||||||
|
else:
|
||||||
|
decision = "conditional"
|
||||||
|
summary = "部分指标达标,存在风险项,建议修复后复测"
|
||||||
|
|
||||||
|
return GoNoGoVerdict(decision, summary, results)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.3 三级结论语义
|
||||||
|
|
||||||
|
| 结论 | 条件 | 含义 |
|
||||||
|
|------|------|------|
|
||||||
|
| **go** | 全部指标达标 | 建议上线 |
|
||||||
|
| **no_go** | 核心指标(通过率/可用性)未达标 | 不建议上线 |
|
||||||
|
| **conditional** | 核心达标但非核心指标(延迟等)有风险 | 建议修复后复测 |
|
||||||
|
|
||||||
|
### 2.4 报告渲染集成
|
||||||
|
|
||||||
|
**HTML 报告**:在 summary 卡片区域上方新增一个醒目的结论横幅:
|
||||||
|
- go: 绿色背景 "GO - 建议上线"
|
||||||
|
- no_go: 红色背景 "NO-GO - 不建议上线"
|
||||||
|
- conditional: 黄色背景 "CONDITIONAL - 存在风险"
|
||||||
|
|
||||||
|
下方增加一个"验收标准比对表",逐条列出阈值、实际值、是否达标。
|
||||||
|
|
||||||
|
**Markdown 报告**:在汇总表格后增加:
|
||||||
|
```markdown
|
||||||
|
## 上线验收结论
|
||||||
|
|
||||||
|
**结论**: GO / NO-GO / CONDITIONAL
|
||||||
|
|
||||||
|
| 指标 | 阈值 | 实际值 | 结果 |
|
||||||
|
|------|------|--------|------|
|
||||||
|
| 判定型通过率 | >= 95% | 96.2% | PASS |
|
||||||
|
| 平均延迟 | <= 5000ms | 3200ms | PASS |
|
||||||
|
```
|
||||||
|
|
||||||
|
**JSON 报告**:在顶层新增 `go_no_go` 字段,包含完整的 `GoNoGoVerdict` 结构。
|
||||||
|
|
||||||
|
### 2.5 API 设计
|
||||||
|
|
||||||
|
**方案 A(推荐)**:在现有报告端点中自动附带
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/reports/{run_id}
|
||||||
|
-> { ...existing report..., go_no_go: { decision, summary, criteria_results } }
|
||||||
|
```
|
||||||
|
|
||||||
|
优点:前端无需改动调用逻辑,结论随报告自动返回。
|
||||||
|
缺点:需要知道使用哪套阈值 -> 从 Scenario 或全局配置读取。
|
||||||
|
|
||||||
|
**方案 B**:独立端点
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/reports/{run_id}/verdict?pass_rate_min=0.95&latency_max=5000
|
||||||
|
```
|
||||||
|
|
||||||
|
优点:阈值灵活,可按需传入。
|
||||||
|
缺点:增加前端调用复杂度。
|
||||||
|
|
||||||
|
**推荐方案 A**,因为阈值应该在创建评测时就确定(绑定到 Scenario 或全局配置),而非每次查看报告时指定。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 达标阈值的配置方式
|
||||||
|
|
||||||
|
### 3.1 配置层级设计
|
||||||
|
|
||||||
|
```
|
||||||
|
全局默认(Settings)
|
||||||
|
└── 场景级覆盖(Scenario.acceptance_criteria)
|
||||||
|
└── 运行级覆盖(EvalRun.acceptance_criteria,可选)
|
||||||
|
```
|
||||||
|
|
||||||
|
**优先级**:运行级 > 场景级 > 全局默认
|
||||||
|
|
||||||
|
### 3.2 具体实现位置
|
||||||
|
|
||||||
|
| 层级 | 存储位置 | 配置方式 |
|
||||||
|
|------|---------|---------|
|
||||||
|
| **全局默认** | `config/settings.py` 新增字段 | 环境变量 `AGENTEVAL_DEFAULT_PASS_RATE_MIN=0.95` |
|
||||||
|
| **场景级** | `Scenario` 模型新增 `acceptance_criteria: Optional[AcceptanceCriteria]` | API/前端创建场景时配置 |
|
||||||
|
| **运行级** | `EvalRun` 模型新增 `acceptance_criteria: Optional[AcceptanceCriteria]`(或从 Scenario 继承) | 创建 run 时可选覆盖 |
|
||||||
|
|
||||||
|
### 3.3 全局默认配置示例
|
||||||
|
|
||||||
|
```python
|
||||||
|
# settings.py 新增
|
||||||
|
default_acceptance_criteria: dict[str, Any] = Field(
|
||||||
|
default_factory=lambda: {
|
||||||
|
"judged_pass_rate_min": 0.95,
|
||||||
|
"pass_rate_min": 0.90,
|
||||||
|
},
|
||||||
|
description="Default acceptance criteria for go/no-go verdicts.",
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.4 场景级配置示例
|
||||||
|
|
||||||
|
在 Scenario 模型中新增可选字段:
|
||||||
|
|
||||||
|
```python
|
||||||
|
class Scenario(BaseModel):
|
||||||
|
# ... existing fields ...
|
||||||
|
acceptance_criteria: Optional[dict[str, Any]] = None
|
||||||
|
# 例如: {"judged_pass_rate_min": 0.98, "avg_latency_max_ms": 3000}
|
||||||
|
```
|
||||||
|
|
||||||
|
前端在场景编辑页增加"验收标准"配置区域(折叠面板),默认展示全局默认值,用户可覆盖。
|
||||||
|
|
||||||
|
### 3.5 数据库迁移
|
||||||
|
|
||||||
|
需要 Alembic 迁移脚本为 `scenarios` 表和 `eval_runs` 表添加 `acceptance_criteria` JSON 列。SQLite 的 `render_as_batch=True` 已配置,迁移无特殊障碍。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 工作量估算
|
||||||
|
|
||||||
|
### 4.1 后端
|
||||||
|
|
||||||
|
| 任务 | 人天 | 说明 |
|
||||||
|
|------|------|------|
|
||||||
|
| `evaluation/go_no_go.py` 核心判定逻辑 | 0.5 | 纯函数,单测覆盖 |
|
||||||
|
| `AcceptanceCriteria` / `GoNoGoVerdict` 数据模型 | 0.5 | Pydantic 模型 |
|
||||||
|
| `RunSummary` / `Scenario` 模型扩展 | 0.5 | 新增字段 + 兼容处理 |
|
||||||
|
| `generate_report()` 集成 go_no_go 结论 | 0.5 | 在报告 dict 中附加 verdict |
|
||||||
|
| `report_render.py` 三种格式渲染 | 1.0 | HTML 横幅 + Markdown 表格 + JSON 字段 |
|
||||||
|
| `settings.py` 全局默认配置 | 0.25 | 新增 Settings 字段 |
|
||||||
|
| Alembic 迁移 | 0.25 | scenarios + eval_runs 加列 |
|
||||||
|
| API 端点调整 | 0.5 | reports router 附带 verdict |
|
||||||
|
| 单元测试 + 集成测试 | 1.0 | go_no_go 逻辑 + API 覆盖 |
|
||||||
|
| **后端小计** | **5.0** | |
|
||||||
|
|
||||||
|
### 4.2 前端
|
||||||
|
|
||||||
|
| 任务 | 人天 | 说明 |
|
||||||
|
|------|------|------|
|
||||||
|
| 场景编辑页增加"验收标准"配置 | 1.0 | FormDrawer 内新增折叠面板 |
|
||||||
|
| Run 报告详情页展示 go/no-go 结论 | 1.0 | 结论横幅 + 比对表格 |
|
||||||
|
| 活动报告页展示活动级结论 | 0.5 | 复用组件 |
|
||||||
|
| **前端小计** | **2.5** | |
|
||||||
|
|
||||||
|
### 4.3 总计
|
||||||
|
|
||||||
|
| 模块 | 人天 |
|
||||||
|
|------|------|
|
||||||
|
| 后端 | 5.0 |
|
||||||
|
| 前端 | 2.5 |
|
||||||
|
| 联调 + 部署 | 0.5 |
|
||||||
|
| **总计** | **8.0 人天** |
|
||||||
|
|
||||||
|
如果只做后端(API 返回 verdict,前端后续迭代),可压缩到 **5.0 人天**。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 推荐方案
|
||||||
|
|
||||||
|
### 5.1 推荐:方案 A + 三级配置 + 渐进式交付
|
||||||
|
|
||||||
|
**理由**:
|
||||||
|
|
||||||
|
1. **纯函数核心**(`go_no_go.py`):与现有架构一致(`metrics.py` / `run_summary.py` 都是纯函数),易测试、易扩展。
|
||||||
|
|
||||||
|
2. **三级阈值配置**(全局默认 -> 场景级 -> 运行级):
|
||||||
|
- 全局默认保证开箱即用,不需要每个场景都配置
|
||||||
|
- 场景级覆盖满足差异化需求(如关键场景要求 98% 通过率)
|
||||||
|
- 运行级覆盖保留灵活性(如临时加严测试)
|
||||||
|
|
||||||
|
3. **自动附带而非独立端点**:前端零改动即可获得 verdict 数据,降低集成成本。
|
||||||
|
|
||||||
|
4. **三级结论(go/no_go/conditional)**:比二元判定更实用。"conditional" 覆盖了"通过率达标但延迟偏高"这类常见场景,避免误判。
|
||||||
|
|
||||||
|
5. **复用现有指标**:不需要新增数据采集,`judged_pass_rate` / `pass_rate` / `avg_latency_ms` / `availability` 均已由引擎计算并持久化。go/no-go 只是在读路径上增加一层阈值比对。
|
||||||
|
|
||||||
|
### 5.2 实施路径
|
||||||
|
|
||||||
|
```
|
||||||
|
Phase 1(MVP,5 人天):
|
||||||
|
- go_no_go.py 核心逻辑
|
||||||
|
- 全局默认阈值(Settings)
|
||||||
|
- generate_report() 集成
|
||||||
|
- JSON/Markdown 渲染
|
||||||
|
- 单元测试
|
||||||
|
|
||||||
|
Phase 2(完善,3 人天):
|
||||||
|
- 场景级 acceptance_criteria 配置
|
||||||
|
- HTML 渲染(结论横幅)
|
||||||
|
- 前端场景编辑页配置面板
|
||||||
|
- 前端报告展示页结论展示
|
||||||
|
|
||||||
|
Phase 3(可选增强):
|
||||||
|
- 活动级 go/no-go(跨 run 聚合判定)
|
||||||
|
- 智能评估报告的评分阈值集成
|
||||||
|
- Webhook 推送 verdict(CI/CD 集成)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.3 关键决策点
|
||||||
|
|
||||||
|
| 决策 | 推荐 | 备选 | 理由 |
|
||||||
|
|------|------|------|------|
|
||||||
|
| 核心指标选择 | `judged_pass_rate` 为主 | `pass_rate` | 连通用例无判定意义,`judged_pass_rate` 更准确反映质量 |
|
||||||
|
| 阈值存储 | JSON 列(灵活) | 独立表(规范化) | 阈值结构简单且固定,JSON 足够,避免过度设计 |
|
||||||
|
| 结论渲染位置 | 报告顶部横幅 | 报告底部 | 结论应第一时间可见,类似体检报告的"总结" |
|
||||||
|
| 活动级 go/no-go | Phase 3 再做 | 同期实现 | 活动级需要跨 run 聚合,复杂度较高,且 ticket #20 聚焦单次验收 |
|
||||||
|
|
||||||
|
### 5.4 与现有架构的契合度
|
||||||
|
|
||||||
|
- **ADR-0002(通过率口径)**:go/no-go 直接消费 `judged_pass_rate`,口径一致
|
||||||
|
- **ADR-0004(聚合口径)**:活动级聚合复用 `aggregate_runs`,不重算
|
||||||
|
- **规则注册表模式**:go_no_go 判定器可设计为可扩展的(未来可能增加新指标)
|
||||||
|
- **纯函数渲染**:遵循 `report_render.py` 的 dict-in/string-out 模式
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 附录:关键代码路径
|
||||||
|
|
||||||
|
| 文件 | 职责 |
|
||||||
|
|------|------|
|
||||||
|
| `backend/agenteval/evaluation/report.py` | 报告生成(`generate_report` / `generate_campaign_report`) |
|
||||||
|
| `backend/agenteval/evaluation/report_render.py` | 报告渲染(HTML/Markdown/JSON) |
|
||||||
|
| `backend/agenteval/evaluation/run_summary.py` | 运行汇总(`build_run_summary`) |
|
||||||
|
| `backend/agenteval/evaluation/metrics.py` | 跨运行聚合(`aggregate_runs`) |
|
||||||
|
| `backend/agenteval/evaluation/judgement.py` | 用例判定(`combine_case_outcome`) |
|
||||||
|
| `backend/agenteval/evaluation/case_verdict.py` | 用例判定读路径(`resolve_case_verdicts`) |
|
||||||
|
| `backend/agenteval/models.py` | 数据模型(`RunSummary` / `Scenario` / `Case`) |
|
||||||
|
| `backend/agenteval/config/settings.py` | 全局配置 |
|
||||||
|
| `backend/agenteval/web/routers/reports.py` | 报告 API 端点 |
|
||||||
138
tests/unit/test_go_no_go.py
Normal file
138
tests/unit/test_go_no_go.py
Normal file
@ -0,0 +1,138 @@
|
|||||||
|
"""Tests for go/no-go acceptance verdict."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from agenteval.evaluation.go_no_go import (
|
||||||
|
AcceptanceCriteria,
|
||||||
|
CriterionResult,
|
||||||
|
GoNoGoVerdict,
|
||||||
|
evaluate_go_no_go,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_go_verdict_all_pass():
|
||||||
|
"""All criteria met -> go."""
|
||||||
|
summary = {
|
||||||
|
"judged_pass_rate": 0.96,
|
||||||
|
"pass_rate": 0.95,
|
||||||
|
"avg_latency_ms": 2000,
|
||||||
|
}
|
||||||
|
criteria = AcceptanceCriteria(
|
||||||
|
judged_pass_rate_min=0.95,
|
||||||
|
pass_rate_min=0.90,
|
||||||
|
avg_latency_max_ms=5000,
|
||||||
|
)
|
||||||
|
verdict = evaluate_go_no_go(summary, criteria)
|
||||||
|
assert verdict.decision == "go"
|
||||||
|
assert "达标" in verdict.summary
|
||||||
|
assert all(r.passed for r in verdict.criteria_results)
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_go_verdict_core_failed():
|
||||||
|
"""Core metric (pass rate) failed -> no_go."""
|
||||||
|
summary = {
|
||||||
|
"judged_pass_rate": 0.80,
|
||||||
|
"pass_rate": 0.75,
|
||||||
|
}
|
||||||
|
criteria = AcceptanceCriteria(
|
||||||
|
judged_pass_rate_min=0.95,
|
||||||
|
pass_rate_min=0.90,
|
||||||
|
)
|
||||||
|
verdict = evaluate_go_no_go(summary, criteria)
|
||||||
|
assert verdict.decision == "no_go"
|
||||||
|
assert "不建议上线" in verdict.summary
|
||||||
|
assert any(not r.passed for r in verdict.criteria_results)
|
||||||
|
|
||||||
|
|
||||||
|
def test_conditional_verdict_non_core_risk():
|
||||||
|
"""Core passed but non-core (latency) failed -> conditional."""
|
||||||
|
summary = {
|
||||||
|
"judged_pass_rate": 0.96,
|
||||||
|
"pass_rate": 0.95,
|
||||||
|
"avg_latency_ms": 8000,
|
||||||
|
}
|
||||||
|
criteria = AcceptanceCriteria(
|
||||||
|
judged_pass_rate_min=0.95,
|
||||||
|
pass_rate_min=0.90,
|
||||||
|
avg_latency_max_ms=5000,
|
||||||
|
)
|
||||||
|
verdict = evaluate_go_no_go(summary, criteria)
|
||||||
|
assert verdict.decision == "conditional"
|
||||||
|
assert "风险" in verdict.summary
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_criteria():
|
||||||
|
"""Default criteria should be applied when None."""
|
||||||
|
summary = {
|
||||||
|
"judged_pass_rate": 0.96,
|
||||||
|
"pass_rate": 0.95,
|
||||||
|
}
|
||||||
|
verdict = evaluate_go_no_go(summary, None)
|
||||||
|
assert verdict.decision == "go"
|
||||||
|
assert len(verdict.criteria_results) >= 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_summary():
|
||||||
|
"""Empty summary -> conditional with no results."""
|
||||||
|
verdict = evaluate_go_no_go({}, None)
|
||||||
|
assert verdict.decision == "conditional"
|
||||||
|
assert "无可用指标" in verdict.summary
|
||||||
|
assert len(verdict.criteria_results) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_criterion_result_model():
|
||||||
|
"""CriterionResult model should work correctly."""
|
||||||
|
result = CriterionResult(
|
||||||
|
criterion="pass_rate",
|
||||||
|
threshold=0.95,
|
||||||
|
actual=0.96,
|
||||||
|
passed=True,
|
||||||
|
detail="通过率 96% >= 95%",
|
||||||
|
)
|
||||||
|
assert result.criterion == "pass_rate"
|
||||||
|
assert result.passed is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_verdict_model():
|
||||||
|
"""GoNoGoVerdict model should work correctly."""
|
||||||
|
verdict = GoNoGoVerdict(
|
||||||
|
decision="go",
|
||||||
|
summary="测试通过",
|
||||||
|
criteria_results=[],
|
||||||
|
)
|
||||||
|
assert verdict.decision == "go"
|
||||||
|
assert verdict.generated_at is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_latency_only_when_configured():
|
||||||
|
"""Latency should only be checked when avg_latency_max_ms is set."""
|
||||||
|
summary = {
|
||||||
|
"judged_pass_rate": 0.96,
|
||||||
|
"avg_latency_ms": 10000, # High latency
|
||||||
|
}
|
||||||
|
# Without latency threshold
|
||||||
|
criteria_no_latency = AcceptanceCriteria(judged_pass_rate_min=0.95)
|
||||||
|
verdict1 = evaluate_go_no_go(summary, criteria_no_latency)
|
||||||
|
assert verdict1.decision == "go"
|
||||||
|
assert all(r.criterion != "avg_latency_ms" for r in verdict1.criteria_results)
|
||||||
|
|
||||||
|
# With latency threshold
|
||||||
|
criteria_with_latency = AcceptanceCriteria(
|
||||||
|
judged_pass_rate_min=0.95,
|
||||||
|
avg_latency_max_ms=5000,
|
||||||
|
)
|
||||||
|
verdict2 = evaluate_go_no_go(summary, criteria_with_latency)
|
||||||
|
assert verdict2.decision == "conditional"
|
||||||
|
assert any(r.criterion == "avg_latency_ms" for r in verdict2.criteria_results)
|
||||||
|
|
||||||
|
|
||||||
|
def test_judged_pass_rate_fallback_to_pass_rate():
|
||||||
|
"""If judged_pass_rate is missing, fall back to pass_rate."""
|
||||||
|
summary = {"pass_rate": 0.96}
|
||||||
|
criteria = AcceptanceCriteria(judged_pass_rate_min=0.95)
|
||||||
|
verdict = evaluate_go_no_go(summary, criteria)
|
||||||
|
assert verdict.decision == "go"
|
||||||
|
# Should have one result using pass_rate as judged_pass_rate
|
||||||
|
assert len(verdict.criteria_results) == 1
|
||||||
|
assert verdict.criteria_results[0].criterion == "judged_pass_rate"
|
||||||
|
assert verdict.criteria_results[0].actual == 0.96
|
||||||
Loading…
Reference in New Issue
Block a user