770d260750
Some checks failed
CI / test (push) Failing after 39s
对比报告可比性收紧为同场景同考纲版本（ADR-0001）：跨版本 API 返回 400 （detail 含双方版本号），报告生成层抛 ValueError；前端对比候选按 同场景 + 同版本过滤，A 变更后自动清空不可比的 B。文档"尚未实现"标注移除。
202 lines
6.6 KiB
Python
202 lines
6.6 KiB
Python
"""Integration tests for the reports API: /api/reports/{id}, /markdown, /compare."""
|
||
|
||
import json
|
||
import pytest
|
||
from fastapi.testclient import TestClient
|
||
from sqlmodel import Session, SQLModel, create_engine
|
||
|
||
from agenteval.models import (
|
||
Case, CaseType, EvalResult, EvalRun, EvalTarget, RunStatus, Scenario,
|
||
PlatformType, ChannelType, TargetStatus, Turn,
|
||
)
|
||
from agenteval.storage.repository import ResultRepository, RunRepository, ScenarioRepository, TargetRepository
|
||
from agenteval.web.app import app
|
||
from agenteval.web.deps import get_db
|
||
|
||
|
||
@pytest.fixture()
|
||
def client_with_db(tmp_path):
|
||
from agenteval.storage.db import ( # noqa: F401
|
||
EvalResultDB, EvalRunDB, EvalTargetDB, FileCategoryDB, FileRecordDB, ScenarioDB, TurnDB,
|
||
)
|
||
engine = create_engine(
|
||
f"sqlite:///{tmp_path / 'reports_api.db'}",
|
||
connect_args={"check_same_thread": False},
|
||
)
|
||
SQLModel.metadata.create_all(engine)
|
||
session = Session(engine)
|
||
|
||
def override_get_db():
|
||
try:
|
||
yield session
|
||
finally:
|
||
pass
|
||
|
||
app.dependency_overrides[get_db] = override_get_db
|
||
client = TestClient(app)
|
||
yield client, session
|
||
app.dependency_overrides.clear()
|
||
session.close()
|
||
|
||
|
||
def _seed_run(
|
||
session: Session,
|
||
name_suffix: str = "",
|
||
scenario_id: str | None = None,
|
||
scenario_version: int = 1,
|
||
) -> str:
|
||
target = EvalTarget(
|
||
name=f"target{name_suffix}",
|
||
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
||
channel_type=ChannelType.TUTU_API,
|
||
channel_config={},
|
||
status=TargetStatus.ACTIVE,
|
||
)
|
||
target = TargetRepository(session).create(target)
|
||
|
||
if scenario_id is None:
|
||
scenario = Scenario(
|
||
name=f"scenario{name_suffix}",
|
||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||
)
|
||
scenario = ScenarioRepository(session).create(scenario)
|
||
scenario_id = scenario.id
|
||
|
||
run = EvalRun(
|
||
target_id=target.id,
|
||
scenario_id=scenario_id,
|
||
scenario_version=scenario_version,
|
||
status=RunStatus.COMPLETED,
|
||
)
|
||
run = RunRepository(session).create(run)
|
||
|
||
result_repo = ResultRepository(session)
|
||
run_repo = RunRepository(session)
|
||
|
||
turn = Turn(
|
||
run_id=run.id, case_id="c1", round_index=1,
|
||
sent_message={"msgBody": {"content": "问题"}},
|
||
reply={"msgBody": {"content": "回答"}},
|
||
latency_ms=300,
|
||
)
|
||
result_repo.save_turn(turn)
|
||
db_turn = run_repo.get_turns(run.id)[0]
|
||
|
||
result_repo.save_result(EvalResult(
|
||
run_id=run.id, case_id="c1",
|
||
turn_id=db_turn.id or "",
|
||
rule_type="response_time",
|
||
passed=True, score=0.9, reason="快",
|
||
))
|
||
|
||
run.summary = {
|
||
"total_cases": 1, "passed_cases": 1, "failed_cases": 0,
|
||
"total_rules": 1, "passed_rules": 1, "pass_rate": 1.0,
|
||
}
|
||
RunRepository(session).update(run)
|
||
return run.id
|
||
|
||
|
||
def test_get_report_200(client_with_db):
|
||
client, session = client_with_db
|
||
run_id = _seed_run(session)
|
||
resp = client.get(f"/api/reports/{run_id}")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["run_id"] == run_id
|
||
assert data["summary"]["pass_rate"] == 1.0
|
||
|
||
|
||
def test_get_report_404(client_with_db):
|
||
client, _ = client_with_db
|
||
resp = client.get("/api/reports/no-such-run")
|
||
assert resp.status_code == 404
|
||
|
||
|
||
def test_get_html_report(client_with_db):
|
||
client, session = client_with_db
|
||
run_id = _seed_run(session)
|
||
resp = client.get(f"/api/reports/{run_id}/html")
|
||
assert resp.status_code == 200
|
||
assert "text/html" in resp.headers["content-type"]
|
||
assert "评测报告" in resp.text
|
||
|
||
|
||
def test_get_json_report(client_with_db):
|
||
client, session = client_with_db
|
||
run_id = _seed_run(session)
|
||
resp = client.get(f"/api/reports/{run_id}/json")
|
||
assert resp.status_code == 200
|
||
data = json.loads(resp.text)
|
||
assert "run_id" in data
|
||
|
||
|
||
def test_get_markdown_report(client_with_db):
|
||
client, session = client_with_db
|
||
run_id = _seed_run(session)
|
||
resp = client.get(f"/api/reports/{run_id}/markdown")
|
||
assert resp.status_code == 200
|
||
assert "text/markdown" in resp.headers["content-type"]
|
||
assert "# 评测报告" in resp.text
|
||
assert "## 汇总" in resp.text
|
||
|
||
|
||
def test_get_markdown_report_attachment_header(client_with_db):
|
||
client, session = client_with_db
|
||
run_id = _seed_run(session)
|
||
resp = client.get(f"/api/reports/{run_id}/markdown")
|
||
assert "attachment" in resp.headers.get("content-disposition", "")
|
||
|
||
|
||
def test_compare_report(client_with_db):
|
||
client, session = client_with_db
|
||
run_id_a = _seed_run(session, "A")
|
||
sid = RunRepository(session).get(run_id_a).scenario_id
|
||
run_id_b = _seed_run(session, "B", scenario_id=sid)
|
||
resp = client.get(f"/api/reports/compare?run1={run_id_a}&run2={run_id_b}")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["run_a"]["run_id"] == run_id_a
|
||
assert data["run_b"]["run_id"] == run_id_b
|
||
assert "delta" in data
|
||
assert "cases" in data
|
||
|
||
|
||
def test_compare_report_different_scenarios_400(client_with_db):
|
||
client, session = client_with_db
|
||
run_id_a = _seed_run(session, "A")
|
||
run_id_b = _seed_run(session, "B") # separate scenario
|
||
resp = client.get(f"/api/reports/compare?run1={run_id_a}&run2={run_id_b}")
|
||
assert resp.status_code == 400
|
||
assert "相同场景" in resp.json()["detail"]
|
||
|
||
|
||
def test_compare_report_cross_version_400(client_with_db):
|
||
"""同场景不同考纲版本 → 400，提示含双方版本号（ticket 05 / ADR-0001）。"""
|
||
client, session = client_with_db
|
||
run_id_a = _seed_run(session, "A")
|
||
sid = RunRepository(session).get(run_id_a).scenario_id
|
||
run_id_b = _seed_run(session, "B", scenario_id=sid, scenario_version=2)
|
||
resp = client.get(f"/api/reports/compare?run1={run_id_a}&run2={run_id_b}")
|
||
assert resp.status_code == 400
|
||
detail = resp.json()["detail"]
|
||
assert "v1" in detail and "v2" in detail
|
||
|
||
|
||
def test_compare_report_same_version_ok(client_with_db):
|
||
client, session = client_with_db
|
||
run_id_a = _seed_run(session, "A", scenario_version=3)
|
||
sid = RunRepository(session).get(run_id_a).scenario_id
|
||
run_id_b = _seed_run(session, "B", scenario_id=sid, scenario_version=3)
|
||
resp = client.get(f"/api/reports/compare?run1={run_id_a}&run2={run_id_b}")
|
||
assert resp.status_code == 200
|
||
assert resp.json()["run_a"]["scenario_version"] == 3
|
||
assert resp.json()["run_b"]["scenario_version"] == 3
|
||
|
||
|
||
def test_compare_report_run_not_found(client_with_db):
|
||
client, session = client_with_db
|
||
run_id = _seed_run(session)
|
||
resp = client.get(f"/api/reports/compare?run1={run_id}&run2=ghost-id")
|
||
assert resp.status_code == 404
|