solahqb/AgentEvalTool
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(run): snapshot scenario version at run creation (ticket 04)

Browse Source 
运行创建时快照场景考纲版本，三种触发来源（手动/AI 助手/CLI）一致；
迁移回填存量运行为其场景当前版本，孤儿运行回填 1。运行列表、
报告头与对比卡片展示 v{n} 版本标签。
This commit is contained in:
sinohqb 2026-07-29 10:59:44 +08:00
parent 43e05ee38d
commit 0a47260237
14 changed files with 170 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.scratch/v0.5/issues/04-run-records-scenario-version.md
View File

@ -4,10 +4,10 @@
**Blocked by:** 03 — 场景版本字段与升版逻辑。
**Status:** ready-for-agent
**Status:** done
- [ ] 新建运行记录 scenario_version = 场景当前版本，三种触发来源一致
- [ ] 迁移回填存量运行；孤儿运行（场景已删）回填 1
- [ ] 运行列表 API 与报告数据含 scenario_version
- [ ] 前端运行列表与报告头展示版本号
- [ ] 集成测试覆盖创建快照与迁移回填（先例：现有运行 API 测试、迁移测试）
- [x] 新建运行记录 scenario_version = 场景当前版本，三种触发来源一致
- [x] 迁移回填存量运行；孤儿运行（场景已删）回填 1
- [x] 运行列表 API 与报告数据含 scenario_version
- [x] 前端运行列表与报告头展示版本号
- [x] 集成测试覆盖创建快照与迁移回填（先例：现有运行 API 测试、迁移测试）

1
backend/agenteval/evaluation/engine.py
View File

@ -118,6 +118,7 @@ class EvalEngine:
id=str(uuid.uuid4()),
target_id=self.target.id or "",
scenario_id=self.scenario.id or "",
scenario_version=self.scenario.version or 1,
status=RunStatus.RUNNING,
triggered_by=self.triggered_by,
started_at=utc_now(),

3
backend/agenteval/evaluation/report.py
View File

@ -172,6 +172,7 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
"target_name": target.name if target else "未知",
"scenario_id": run.scenario_id,
"scenario_name": scenario.name if scenario else "未知",
"scenario_version": run.scenario_version,
"status": run.status.value,
"started_at": iso_utc(run.started_at),
"completed_at": iso_utc(run.completed_at),
@ -241,6 +242,7 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[
"run_id": run_id_1,
"target_name": report_a.get("target_name"),
"scenario_name": report_a.get("scenario_name"),
"scenario_version": report_a.get("scenario_version"),
"status": report_a.get("status"),
"started_at": report_a.get("started_at"),
"summary": report_a["summary"],
@ -249,6 +251,7 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[
"run_id": run_id_2,
"target_name": report_b.get("target_name"),
"scenario_name": report_b.get("scenario_name"),
"scenario_version": report_b.get("scenario_version"),
"status": report_b.get("status"),
"started_at": report_b.get("started_at"),
"summary": report_b["summary"],

2
backend/agenteval/models.py
View File

@ -162,6 +162,8 @@ class EvalRun(BaseModel):
id: Optional[str] = None
target_id: str
scenario_id: str
# 创建时快照的场景考纲版本（ADR-0001）
scenario_version: int = 1
status: RunStatus = RunStatus.PENDING
triggered_by: RunTrigger = RunTrigger.MANUAL
started_at: Optional[datetime] = None

1
backend/agenteval/storage/db.py
View File

@ -168,6 +168,7 @@ class EvalRunDB(SQLModel, table=True):
id: Optional[str] = Field(default_factory=new_uuid, primary_key=True)
target_id: Optional[str] = Field(default=None, foreign_key="eval_targets.id")
scenario_id: Optional[str] = Field(default=None, foreign_key="scenarios.id")
scenario_version: int = Field(default=1)
status: str = "pending"
triggered_by: str = Field(default="manual")
started_at: Optional[datetime] = Field(default_factory=utc_now)

2
backend/agenteval/storage/repository.py
View File

@ -83,6 +83,7 @@ def _run_to_db(run: EvalRun) -> EvalRunDB:
id=run.id,
target_id=run.target_id,
scenario_id=run.scenario_id,
scenario_version=run.scenario_version,
status=run.status.value,
triggered_by=run.triggered_by.value,
started_at=run.started_at,
@ -98,6 +99,7 @@ def _run_from_db(db: EvalRunDB) -> EvalRun:
id=db.id,
target_id=db.target_id,
scenario_id=db.scenario_id,
scenario_version=db.scenario_version or 1,
status=db.status,
triggered_by=db.triggered_by or "manual",
started_at=db.started_at,

1
backend/agenteval/web/routers/runs.py
View File

@ -95,6 +95,7 @@ async def start_run(
run = EvalRun(
target_id=request.target_id,
scenario_id=request.scenario_id,
scenario_version=scenario.version or 1,
triggered_by=request.triggered_by,
)
run = RunRepository(session).create(run)

1
frontend/web/src/api.ts
View File

@ -141,6 +141,7 @@ export interface Run {
id: string
target_id: string
scenario_id: string
scenario_version?: number
status: string
triggered_by?: RunTrigger
scenario_name?: string | null

7
frontend/web/src/components/RunList.tsx
View File

@ -292,6 +292,13 @@ function RunRow({ r, selected, targetName, scenarioName, onSelect, onOpenReport,
<span>{shortDateTime(r.started_at)}</span>
<span>·</span>
<span>{elapsedStr(r.started_at, r.completed_at)}</span>
{r.scenario_version != null && (
<Tooltip title="场景考纲版本">
<Tag style={{ marginLeft: 2, marginRight: 0, fontSize: 10, lineHeight: '16px', padding: '0 4px' }}>
v{r.scenario_version}
</Tag>
</Tooltip>
)}
{r.triggered_by && r.triggered_by !== 'manual' && (
<Tag
color={triggerColors[r.triggered_by] ?? 'default'}

22
frontend/web/src/pages/Reports.tsx
View File

@ -38,6 +38,7 @@ interface Report {
run_id: string
target_name: string
scenario_name: string
scenario_version?: number
status: string
started_at: string
completed_at: string | null
@ -55,8 +56,8 @@ interface Report {
}
interface CompareResult {
run_a: { run_id: string; target_name: string; scenario_name: string; status: string; started_at: string; summary: Report['summary'] }
run_b: { run_id: string; target_name: string; scenario_name: string; status: string; started_at: string; summary: Report['summary'] }
run_a: { run_id: string; target_name: string; scenario_name: string; scenario_version?: number; status: string; started_at: string; summary: Report['summary'] }
run_b: { run_id: string; target_name: string; scenario_name: string; scenario_version?: number; status: string; started_at: string; summary: Report['summary'] }
delta: { pass_rate: number; passed_cases: number; passed_rules: number }
cases: Array<{
case_id: string
@ -379,7 +380,12 @@ function SingleReportView({ report }: { report: Report | null }) {
<Card style={{ marginBottom: 16 }}>
<Descriptions size="small" column={2}>
<Descriptions.Item label="评测对象">{report.target_name}</Descriptions.Item>
<Descriptions.Item label="评测场景">{report.scenario_name}</Descriptions.Item>
<Descriptions.Item label="评测场景">
<Space size={6}>
{report.scenario_name}
{report.scenario_version != null && <Tag color="geekblue">v{report.scenario_version}</Tag>}
</Space>
</Descriptions.Item>
<Descriptions.Item label="开始时间">{formatDateTime(report.started_at)}</Descriptions.Item>
<Descriptions.Item label="完成时间">{report.completed_at ? formatDateTime(report.completed_at) : '-'}</Descriptions.Item>
</Descriptions>
@ -474,7 +480,10 @@ function CompareView({ result }: { result: CompareResult | null }) {
<Col span={11}>
<Card title={<span style={{ color: '#1677ff' }}>报告 A — {run_a.run_id.slice(0, 8)}…</span>} size="small">
<Descriptions size="small" column={1}>
<Descriptions.Item label="场景">{run_a.scenario_name}</Descriptions.Item>
<Descriptions.Item label="场景">
{run_a.scenario_name}
{run_a.scenario_version != null && <Tag color="geekblue" style={{ marginLeft: 6 }}>v{run_a.scenario_version}</Tag>}
</Descriptions.Item>
<Descriptions.Item label="时间">{formatDateTime(run_a.started_at)}</Descriptions.Item>
<Descriptions.Item label="通过率">{(run_a.summary.pass_rate * 100).toFixed(1)}%</Descriptions.Item>
<Descriptions.Item label="用例">{run_a.summary.passed_cases}/{run_a.summary.total_cases}</Descriptions.Item>
@ -492,7 +501,10 @@ function CompareView({ result }: { result: CompareResult | null }) {
<Col span={11}>
<Card title={<span style={{ color: '#52c41a' }}>报告 B — {run_b.run_id.slice(0, 8)}…</span>} size="small">
<Descriptions size="small" column={1}>
<Descriptions.Item label="场景">{run_b.scenario_name}</Descriptions.Item>
<Descriptions.Item label="场景">
{run_b.scenario_name}
{run_b.scenario_version != null && <Tag color="geekblue" style={{ marginLeft: 6 }}>v{run_b.scenario_version}</Tag>}
</Descriptions.Item>
<Descriptions.Item label="时间">{formatDateTime(run_b.started_at)}</Descriptions.Item>
<Descriptions.Item label="通过率">{(run_b.summary.pass_rate * 100).toFixed(1)}%</Descriptions.Item>
<Descriptions.Item label="用例">{run_b.summary.passed_cases}/{run_b.summary.total_cases}</Descriptions.Item>

34
migrations/versions/d5b8c2e4f617_add_scenario_version_to_eval_runs.py Normal file
View File

@ -0,0 +1,34 @@
"""add scenario_version to eval_runs
Revision ID: d5b8c2e4f617
Revises: c8e2f5a7b901
Create Date: 2026-07-29
"""
from typing import Sequence, Union
import sqlalchemy as sa
import sqlmodel # noqa: F401
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "d5b8c2e4f617"
down_revision: Union[str, Sequence[str], None] = "c8e2f5a7b901"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
with op.batch_alter_table("eval_runs") as batch_op:
batch_op.add_column(
sa.Column("scenario_version", sa.Integer(), nullable=False, server_default="1")
)
# 存量运行回填其场景当前版本；孤儿运行（场景已删）保持默认 1
op.execute(
"UPDATE eval_runs SET scenario_version = COALESCE("
"(SELECT version FROM scenarios WHERE scenarios.id = eval_runs.scenario_id), 1)"
)
def downgrade() -> None:
with op.batch_alter_table("eval_runs") as batch_op:
batch_op.drop_column("scenario_version")

27
tests/integration/test_runs_api.py
View File

@ -238,3 +238,30 @@ async def test_list_runs_includes_names_and_trigger(client, seeded_db, mock_chan
assert row["scenario_name"] == "mock-scenario"
assert row["target_name"] == "mock-target"
assert row["triggered_by"] == "ai_assistant"
# ── scenario_version snapshot (ticket 04) ────────────────────────────────
async def test_start_run_snapshots_scenario_version(client, seeded_db, mock_channel):
resp = await client.post("/api/runs", json={
"target_id": "t-1", "scenario_id": "s-1",
})
assert resp.status_code == 200
assert resp.json()["scenario_version"] == 1
# 编辑考纲 → 场景升版 → 新运行快照新版本；旧运行保持 1
old_run_id = resp.json()["id"]
scenario = ScenarioRepository(seeded_db).get("s-1")
scenario.cases.append(Case(id="c2", type=CaseType.SINGLE, messages=["more"]))
ScenarioRepository(seeded_db).update(scenario)
resp2 = await client.post("/api/runs", json={
"target_id": "t-1", "scenario_id": "s-1", "triggered_by": "ai_assistant",
})
assert resp2.json()["scenario_version"] == 2
old = (await client.get(f"/api/runs/{old_run_id}")).json()
assert old["scenario_version"] == 1
listing = (await client.get("/api/runs")).json()
assert {row["scenario_version"] for row in listing} == {1, 2}

50
tests/integration/test_scenario_version_migration.py Normal file
View File

@ -0,0 +1,50 @@
"""Exercise the scenario_version backfill migration against a real SQLite file (ticket 04)."""
from pathlib import Path
from alembic import command
from alembic.config import Config
from sqlalchemy import create_engine, inspect, text
def test_scenario_version_backfill_migration(tmp_path: Path, monkeypatch):
from agenteval.storage import db as db_module
database_url = f"sqlite:///{tmp_path / 'migration.db'}"
monkeypatch.setattr(db_module, "DATABASE_URL", database_url)
config = Config(str(Path(__file__).resolve().parents[2] / "alembic.ini"))
legacy_engine = create_engine(database_url)
with legacy_engine.begin() as connection:
connection.execute(text("CREATE TABLE scenarios (id VARCHAR PRIMARY KEY, name VARCHAR NOT NULL)"))
connection.execute(text("CREATE TABLE eval_results (id VARCHAR PRIMARY KEY, run_id VARCHAR NOT NULL)"))
connection.execute(
text(
"CREATE TABLE eval_runs (id VARCHAR PRIMARY KEY, target_id VARCHAR NOT NULL, "
"scenario_id VARCHAR NOT NULL)"
)
)
connection.execute(text("CREATE TABLE turns (id VARCHAR PRIMARY KEY, run_id VARCHAR NOT NULL)"))
connection.execute(text("INSERT INTO scenarios (id, name) VALUES ('s1', '场景一')"))
connection.execute(
text("INSERT INTO eval_runs (id, target_id, scenario_id) VALUES ('r1', 't1', 's1')")
)
connection.execute(
text("INSERT INTO eval_runs (id, target_id, scenario_id) VALUES ('r2', 't1', 'ghost')")
)
# 先升到场景版本迁移，将 s1 手动升到 5，验证回填按场景当前版本 join
command.upgrade(config, "c8e2f5a7b901")
with create_engine(database_url).begin() as connection:
connection.execute(text("UPDATE scenarios SET version = 5 WHERE id = 's1'"))
command.upgrade(config, "head")
engine = create_engine(database_url)
inspector = inspect(engine)
assert "scenario_version" in {c["name"] for c in inspector.get_columns("eval_runs")}
with engine.connect() as connection:
rows = dict(connection.execute(text("SELECT id, scenario_version FROM eval_runs")).fetchall())
assert rows["r1"] == 5 # 回填为场景当前版本
assert rows["r2"] == 1 # 孤儿运行（场景已删）回填 1

18
tests/unit/test_engine.py
View File

@ -437,3 +437,21 @@ async def test_pure_expectation_case_behavior_unchanged(db_session):
assert run.summary["passed_cases"] == 1
assert run.summary["failed_cases"] == 1
# ── scenario_version snapshot (ticket 04) ────────────────────────────────
async def test_engine_run_snapshots_scenario_version(db_session):
"""引擎直启（CLI 路径）创建的运行快照场景当前版本。"""
scenario = Scenario(
id="s-1", name="versioned", version=3,
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
run = await engine.run()
assert run.scenario_version == 3
persisted = RunRepository(db_session).get(run.id)
assert persisted.scenario_version == 3