diff --git a/.scratch/v0.5/issues/05-compare-same-version-guard.md b/.scratch/v0.5/issues/05-compare-same-version-guard.md index 4e7403a..d95e79e 100644 --- a/.scratch/v0.5/issues/05-compare-same-version-guard.md +++ b/.scratch/v0.5/issues/05-compare-same-version-guard.md @@ -4,10 +4,10 @@ **Blocked by:** 04 — 运行记录场景版本。 -**Status:** ready-for-agent +**Status:** done -- [ ] 同场景不同版本的两次运行对比:API 400,detail 含双方版本号 -- [ ] 同场景同版本对比正常生成(含动态用例场景——同考纲即可比) -- [ ] 报告生成层对跨版本对比抛出明确错误 -- [ ] 前端对比候选按同场景 + 同版本过滤;跨版本被拒时提示可读 -- [ ] 集成与单元测试覆盖拒绝与放行两侧(先例:现有对比报告测试) +- [x] 同场景不同版本的两次运行对比:API 400,detail 含双方版本号 +- [x] 同场景同版本对比正常生成(含动态用例场景——同考纲即可比) +- [x] 报告生成层对跨版本对比抛出明确错误 +- [x] 前端对比候选按同场景 + 同版本过滤;跨版本被拒时提示可读 +- [x] 集成与单元测试覆盖拒绝与放行两侧(先例:现有对比报告测试) diff --git a/CONTEXT.md b/CONTEXT.md index 3e98cd1..b57adab 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -41,7 +41,7 @@ _Avoid_: 测试点、题目 _Avoid_: 差异报告 **场景版本(Scenario Version)**: -场景考纲的版本标识。仅考纲字段(用例集、模型绑定、LLM 配置)变更时递增;名称、描述、标签等元数据编辑不升版。(决策见 ADR-0001,尚未实现) +场景考纲的版本标识。仅考纲字段(用例集、模型绑定、LLM 配置)变更时递增;名称、描述、标签等元数据编辑不升版。(决策见 ADR-0001) _Avoid_: 修订号 **评测运行(Run / EvalRun)**: diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index 398cfed..ebab83e 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -199,6 +199,12 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[ # "changed" and the diff would be meaningless — reject early. if report_a.get("scenario_id") != report_b.get("scenario_id"): raise ValueError("compare report requires both runs to use the same scenario") + # 同场景还须同考纲版本才可比(ADR-0001) + if report_a.get("scenario_version") != report_b.get("scenario_version"): + raise ValueError( + "compare report requires the same scenario version " + f"(A: v{report_a.get('scenario_version')}, B: v{report_b.get('scenario_version')})" + ) def _summary_delta(key: str) -> float: return report_b["summary"][key] - report_a["summary"][key] diff --git a/backend/agenteval/web/routers/reports.py b/backend/agenteval/web/routers/reports.py index ea09dac..8e1ae16 100644 --- a/backend/agenteval/web/routers/reports.py +++ b/backend/agenteval/web/routers/reports.py @@ -31,6 +31,12 @@ def get_compare_report( raise HTTPException(status_code=404, detail=f"run not found: {run2}") if run_a.scenario_id != run_b.scenario_id: raise HTTPException(status_code=400, detail="对比报告要求两个运行使用相同场景") + if run_a.scenario_version != run_b.scenario_version: + raise HTTPException( + status_code=400, + detail=f"对比报告要求相同的场景考纲版本(A: v{run_a.scenario_version}, B: v{run_b.scenario_version})," + "考纲变更前后的运行不可比", + ) return generate_compare_report(run1, run2, session) diff --git a/docs/adr/0001-scenario-versioning-for-comparability.md b/docs/adr/0001-scenario-versioning-for-comparability.md index 0a528c6..41159b1 100644 --- a/docs/adr/0001-scenario-versioning-for-comparability.md +++ b/docs/adr/0001-scenario-versioning-for-comparability.md @@ -10,5 +10,5 @@ ## Consequences -- 数据模型需加 scenario version 字段,Run 需记录所用版本(v0.5 实施项,尚未实现) +- 数据模型已加 scenario version 字段,Run 创建时快照所用版本(v0.5 已实现) - 动态用例每次运行题目不同不影响可比性——可比性单位是"同考纲"(同场景同版本),不是"同考卷" diff --git a/frontend/web/src/pages/Reports.tsx b/frontend/web/src/pages/Reports.tsx index 3adcc52..8c563e4 100644 --- a/frontend/web/src/pages/Reports.tsx +++ b/frontend/web/src/pages/Reports.tsx @@ -125,10 +125,11 @@ export default function ReportsPage() { const handleView = (runId: string) => { setSearchParams({ run: runId }) loadReport(runId) - // 对比报告要求同场景:A 变更后若 B 场景不同则清空 + // 对比报告要求同场景同版本:A 变更后若 B 不可比则清空 const a = runs.find((r) => r.id === runId) const b = runs.find((r) => r.id === compareRunId) - if (a && b && a.scenario_id !== b.scenario_id) setCompareRunId('') + if (a && b && (a.scenario_id !== b.scenario_id + || (a.scenario_version ?? 1) !== (b.scenario_version ?? 1))) setCompareRunId('') } const handleCompare = async () => { @@ -205,10 +206,14 @@ export default function ReportsPage() { const runSelectOptions = filteredRuns.map(buildOption) - // 报告 B 只能选与报告 A 同场景的 run + // 报告 B 只能选与报告 A 同场景同版本(同考纲)的 run const selectedRun = runs.find((r) => r.id === selectedRunId) const compareOptions = runs - .filter((r) => r.id !== selectedRunId && selectedRun && r.scenario_id === selectedRun.scenario_id) + .filter((r) => + r.id !== selectedRunId + && selectedRun + && r.scenario_id === selectedRun.scenario_id + && (r.scenario_version ?? 1) === (selectedRun.scenario_version ?? 1)) .map(buildOption) const optionFilter = (input: string, opt?: { searchText?: string }) => @@ -291,7 +296,7 @@ export default function ReportsPage() { filterOption={optionFilter} options={compareOptions} popupMatchSelectWidth={false} - notFoundContent={} + notFoundContent={} />