From 0a4726023761f8328709acb73f915b93c7d7d9ad Mon Sep 17 00:00:00 2001 From: sinohqb Date: Wed, 29 Jul 2026 10:59:44 +0800 Subject: [PATCH] feat(run): snapshot scenario version at run creation (ticket 04) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 运行创建时快照场景考纲版本，三种触发来源（手动/AI 助手/CLI）一致； 迁移回填存量运行为其场景当前版本，孤儿运行回填 1。运行列表、 报告头与对比卡片展示 v{n} 版本标签。 --- .../issues/04-run-records-scenario-version.md | 12 ++--- backend/agenteval/evaluation/engine.py | 1 + backend/agenteval/evaluation/report.py | 3 ++ backend/agenteval/models.py | 2 + backend/agenteval/storage/db.py | 1 + backend/agenteval/storage/repository.py | 2 + backend/agenteval/web/routers/runs.py | 1 + frontend/web/src/api.ts | 1 + frontend/web/src/components/RunList.tsx | 7 +++ frontend/web/src/pages/Reports.tsx | 22 ++++++-- ...4f617_add_scenario_version_to_eval_runs.py | 34 +++++++++++++ tests/integration/test_runs_api.py | 27 ++++++++++ .../test_scenario_version_migration.py | 50 +++++++++++++++++++ tests/unit/test_engine.py | 18 +++++++ 14 files changed, 170 insertions(+), 11 deletions(-) create mode 100644 migrations/versions/d5b8c2e4f617_add_scenario_version_to_eval_runs.py create mode 100644 tests/integration/test_scenario_version_migration.py diff --git a/.scratch/v0.5/issues/04-run-records-scenario-version.md b/.scratch/v0.5/issues/04-run-records-scenario-version.md index 777be0e..893c2aa 100644 --- a/.scratch/v0.5/issues/04-run-records-scenario-version.md +++ b/.scratch/v0.5/issues/04-run-records-scenario-version.md @@ -4,10 +4,10 @@ **Blocked by:** 03 — 场景版本字段与升版逻辑。 -**Status:** ready-for-agent +**Status:** done -- [ ] 新建运行记录 scenario_version = 场景当前版本，三种触发来源一致 -- [ ] 迁移回填存量运行；孤儿运行（场景已删）回填 1 -- [ ] 运行列表 API 与报告数据含 scenario_version -- [ ] 前端运行列表与报告头展示版本号 -- [ ] 集成测试覆盖创建快照与迁移回填（先例：现有运行 API 测试、迁移测试） +- [x] 新建运行记录 scenario_version = 场景当前版本，三种触发来源一致 +- [x] 迁移回填存量运行；孤儿运行（场景已删）回填 1 +- [x] 运行列表 API 与报告数据含 scenario_version +- [x] 前端运行列表与报告头展示版本号 +- [x] 集成测试覆盖创建快照与迁移回填（先例：现有运行 API 测试、迁移测试） diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index 87de4d9..fd17558 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -118,6 +118,7 @@ class EvalEngine: id=str(uuid.uuid4()), target_id=self.target.id or "", scenario_id=self.scenario.id or "", + scenario_version=self.scenario.version or 1, status=RunStatus.RUNNING, triggered_by=self.triggered_by, started_at=utc_now(), diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index 2d58e6e..398cfed 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -172,6 +172,7 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: "target_name": target.name if target else "未知", "scenario_id": run.scenario_id, "scenario_name": scenario.name if scenario else "未知", + "scenario_version": run.scenario_version, "status": run.status.value, "started_at": iso_utc(run.started_at), "completed_at": iso_utc(run.completed_at), @@ -241,6 +242,7 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[ "run_id": run_id_1, "target_name": report_a.get("target_name"), "scenario_name": report_a.get("scenario_name"), + "scenario_version": report_a.get("scenario_version"), "status": report_a.get("status"), "started_at": report_a.get("started_at"), "summary": report_a["summary"], @@ -249,6 +251,7 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[ "run_id": run_id_2, "target_name": report_b.get("target_name"), "scenario_name": report_b.get("scenario_name"), + "scenario_version": report_b.get("scenario_version"), "status": report_b.get("status"), "started_at": report_b.get("started_at"), "summary": report_b["summary"], diff --git a/backend/agenteval/models.py b/backend/agenteval/models.py index 4783bbe..5eac6ca 100644 --- a/backend/agenteval/models.py +++ b/backend/agenteval/models.py @@ -162,6 +162,8 @@ class EvalRun(BaseModel): id: Optional[str] = None target_id: str scenario_id: str + # 创建时快照的场景考纲版本（ADR-0001） + scenario_version: int = 1 status: RunStatus = RunStatus.PENDING triggered_by: RunTrigger = RunTrigger.MANUAL started_at: Optional[datetime] = None diff --git a/backend/agenteval/storage/db.py b/backend/agenteval/storage/db.py index 6b83890..0c615df 100644 --- a/backend/agenteval/storage/db.py +++ b/backend/agenteval/storage/db.py @@ -168,6 +168,7 @@ class EvalRunDB(SQLModel, table=True): id: Optional[str] = Field(default_factory=new_uuid, primary_key=True) target_id: Optional[str] = Field(default=None, foreign_key="eval_targets.id") scenario_id: Optional[str] = Field(default=None, foreign_key="scenarios.id") + scenario_version: int = Field(default=1) status: str = "pending" triggered_by: str = Field(default="manual") started_at: Optional[datetime] = Field(default_factory=utc_now) diff --git a/backend/agenteval/storage/repository.py b/backend/agenteval/storage/repository.py index da4ae1e..442714c 100644 --- a/backend/agenteval/storage/repository.py +++ b/backend/agenteval/storage/repository.py @@ -83,6 +83,7 @@ def _run_to_db(run: EvalRun) -> EvalRunDB: id=run.id, target_id=run.target_id, scenario_id=run.scenario_id, + scenario_version=run.scenario_version, status=run.status.value, triggered_by=run.triggered_by.value, started_at=run.started_at, @@ -98,6 +99,7 @@ def _run_from_db(db: EvalRunDB) -> EvalRun: id=db.id, target_id=db.target_id, scenario_id=db.scenario_id, + scenario_version=db.scenario_version or 1, status=db.status, triggered_by=db.triggered_by or "manual", started_at=db.started_at, diff --git a/backend/agenteval/web/routers/runs.py b/backend/agenteval/web/routers/runs.py index 89d8ee4..6a17b29 100644 --- a/backend/agenteval/web/routers/runs.py +++ b/backend/agenteval/web/routers/runs.py @@ -95,6 +95,7 @@ async def start_run( run = EvalRun( target_id=request.target_id, scenario_id=request.scenario_id, + scenario_version=scenario.version or 1, triggered_by=request.triggered_by, ) run = RunRepository(session).create(run) diff --git a/frontend/web/src/api.ts b/frontend/web/src/api.ts index 9f056a3..53f3d1a 100644 --- a/frontend/web/src/api.ts +++ b/frontend/web/src/api.ts @@ -141,6 +141,7 @@ export interface Run { id: string target_id: string scenario_id: string + scenario_version?: number status: string triggered_by?: RunTrigger scenario_name?: string | null diff --git a/frontend/web/src/components/RunList.tsx b/frontend/web/src/components/RunList.tsx index 83577fd..df09596 100644 --- a/frontend/web/src/components/RunList.tsx +++ b/frontend/web/src/components/RunList.tsx @@ -292,6 +292,13 @@ function RunRow({ r, selected, targetName, scenarioName, onSelect, onOpenReport, {shortDateTime(r.started_at)} · {elapsedStr(r.started_at, r.completed_at)} + {r.scenario_version != null && ( + + + v{r.scenario_version} + + + )} {r.triggered_by && r.triggered_by !== 'manual' && ( {report.target_name} - {report.scenario_name} + + + {report.scenario_name} + {report.scenario_version != null && v{report.scenario_version}} + + {formatDateTime(report.started_at)} {report.completed_at ? formatDateTime(report.completed_at) : '-'} @@ -474,7 +480,10 @@ function CompareView({ result }: { result: CompareResult | null }) { 报告 A — {run_a.run_id.slice(0, 8)}…} size="small"> - {run_a.scenario_name} + + {run_a.scenario_name} + {run_a.scenario_version != null && v{run_a.scenario_version}} + {formatDateTime(run_a.started_at)} {(run_a.summary.pass_rate * 100).toFixed(1)}% {run_a.summary.passed_cases}/{run_a.summary.total_cases} @@ -492,7 +501,10 @@ function CompareView({ result }: { result: CompareResult | null }) { 报告 B — {run_b.run_id.slice(0, 8)}…} size="small"> - {run_b.scenario_name} + + {run_b.scenario_name} + {run_b.scenario_version != null && v{run_b.scenario_version}} + {formatDateTime(run_b.started_at)} {(run_b.summary.pass_rate * 100).toFixed(1)}% {run_b.summary.passed_cases}/{run_b.summary.total_cases} diff --git a/migrations/versions/d5b8c2e4f617_add_scenario_version_to_eval_runs.py b/migrations/versions/d5b8c2e4f617_add_scenario_version_to_eval_runs.py new file mode 100644 index 0000000..aedb58d --- /dev/null +++ b/migrations/versions/d5b8c2e4f617_add_scenario_version_to_eval_runs.py @@ -0,0 +1,34 @@ +"""add scenario_version to eval_runs + +Revision ID: d5b8c2e4f617 +Revises: c8e2f5a7b901 +Create Date: 2026-07-29 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +import sqlmodel # noqa: F401 +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "d5b8c2e4f617" +down_revision: Union[str, Sequence[str], None] = "c8e2f5a7b901" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table("eval_runs") as batch_op: + batch_op.add_column( + sa.Column("scenario_version", sa.Integer(), nullable=False, server_default="1") + ) + # 存量运行回填其场景当前版本；孤儿运行（场景已删）保持默认 1 + op.execute( + "UPDATE eval_runs SET scenario_version = COALESCE(" + "(SELECT version FROM scenarios WHERE scenarios.id = eval_runs.scenario_id), 1)" + ) + + +def downgrade() -> None: + with op.batch_alter_table("eval_runs") as batch_op: + batch_op.drop_column("scenario_version") diff --git a/tests/integration/test_runs_api.py b/tests/integration/test_runs_api.py index 3954af7..ae8f540 100644 --- a/tests/integration/test_runs_api.py +++ b/tests/integration/test_runs_api.py @@ -238,3 +238,30 @@ async def test_list_runs_includes_names_and_trigger(client, seeded_db, mock_chan assert row["scenario_name"] == "mock-scenario" assert row["target_name"] == "mock-target" assert row["triggered_by"] == "ai_assistant" + + +# ── scenario_version snapshot (ticket 04) ──────────────────────────────── + +async def test_start_run_snapshots_scenario_version(client, seeded_db, mock_channel): + resp = await client.post("/api/runs", json={ + "target_id": "t-1", "scenario_id": "s-1", + }) + assert resp.status_code == 200 + assert resp.json()["scenario_version"] == 1 + + # 编辑考纲 → 场景升版 → 新运行快照新版本；旧运行保持 1 + old_run_id = resp.json()["id"] + scenario = ScenarioRepository(seeded_db).get("s-1") + scenario.cases.append(Case(id="c2", type=CaseType.SINGLE, messages=["more"])) + ScenarioRepository(seeded_db).update(scenario) + + resp2 = await client.post("/api/runs", json={ + "target_id": "t-1", "scenario_id": "s-1", "triggered_by": "ai_assistant", + }) + assert resp2.json()["scenario_version"] == 2 + + old = (await client.get(f"/api/runs/{old_run_id}")).json() + assert old["scenario_version"] == 1 + + listing = (await client.get("/api/runs")).json() + assert {row["scenario_version"] for row in listing} == {1, 2} diff --git a/tests/integration/test_scenario_version_migration.py b/tests/integration/test_scenario_version_migration.py new file mode 100644 index 0000000..4d98042 --- /dev/null +++ b/tests/integration/test_scenario_version_migration.py @@ -0,0 +1,50 @@ +"""Exercise the scenario_version backfill migration against a real SQLite file (ticket 04).""" + +from pathlib import Path + +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, inspect, text + + +def test_scenario_version_backfill_migration(tmp_path: Path, monkeypatch): + from agenteval.storage import db as db_module + + database_url = f"sqlite:///{tmp_path / 'migration.db'}" + monkeypatch.setattr(db_module, "DATABASE_URL", database_url) + config = Config(str(Path(__file__).resolve().parents[2] / "alembic.ini")) + + legacy_engine = create_engine(database_url) + with legacy_engine.begin() as connection: + connection.execute(text("CREATE TABLE scenarios (id VARCHAR PRIMARY KEY, name VARCHAR NOT NULL)")) + connection.execute(text("CREATE TABLE eval_results (id VARCHAR PRIMARY KEY, run_id VARCHAR NOT NULL)")) + connection.execute( + text( + "CREATE TABLE eval_runs (id VARCHAR PRIMARY KEY, target_id VARCHAR NOT NULL, " + "scenario_id VARCHAR NOT NULL)" + ) + ) + connection.execute(text("CREATE TABLE turns (id VARCHAR PRIMARY KEY, run_id VARCHAR NOT NULL)")) + connection.execute(text("INSERT INTO scenarios (id, name) VALUES ('s1', '场景一')")) + connection.execute( + text("INSERT INTO eval_runs (id, target_id, scenario_id) VALUES ('r1', 't1', 's1')") + ) + connection.execute( + text("INSERT INTO eval_runs (id, target_id, scenario_id) VALUES ('r2', 't1', 'ghost')") + ) + + # 先升到场景版本迁移，将 s1 手动升到 5，验证回填按场景当前版本 join + command.upgrade(config, "c8e2f5a7b901") + with create_engine(database_url).begin() as connection: + connection.execute(text("UPDATE scenarios SET version = 5 WHERE id = 's1'")) + + command.upgrade(config, "head") + + engine = create_engine(database_url) + inspector = inspect(engine) + assert "scenario_version" in {c["name"] for c in inspector.get_columns("eval_runs")} + + with engine.connect() as connection: + rows = dict(connection.execute(text("SELECT id, scenario_version FROM eval_runs")).fetchall()) + assert rows["r1"] == 5 # 回填为场景当前版本 + assert rows["r2"] == 1 # 孤儿运行（场景已删）回填 1 diff --git a/tests/unit/test_engine.py b/tests/unit/test_engine.py index 9dd79bb..241d040 100644 --- a/tests/unit/test_engine.py +++ b/tests/unit/test_engine.py @@ -437,3 +437,21 @@ async def test_pure_expectation_case_behavior_unchanged(db_session): assert run.summary["passed_cases"] == 1 assert run.summary["failed_cases"] == 1 + +# ── scenario_version snapshot (ticket 04) ──────────────────────────────── + +async def test_engine_run_snapshots_scenario_version(db_session): + """引擎直启（CLI 路径）创建的运行快照场景当前版本。""" + scenario = Scenario( + id="s-1", name="versioned", version=3, + cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])], + ) + channel = MockChannel() + engine = _build_engine(scenario, channel, session=db_session) + + run = await engine.run() + + assert run.scenario_version == 3 + persisted = RunRepository(db_session).get(run.id) + assert persisted.scenario_version == 3 +