5db0ede4f4
Some checks failed
CI / test (push) Failing after 50s
「用例是否通过」此前散落 8 处且互相矛盾：engine 权威判定焊死在持久化里 不可单测；report 聚合/compare/markdown 各自从规则结果反推，规则还不一致 （markdown 用 all([]) 把故障用例误渲染成 ✅）。 - 新增纯函数 evaluation/judgement.combine_case_outcome（RuleOutcome/ CaseOutcome），判定组合脱离通道与 DB 可单测（判定矩阵 14 例） - engine 调用它一次，逐用例权威结果写入 summary.case_outcomes（JSON， 零迁移）；report/compare/markdown 只读权威值，老 run fallback 反推 - 故障用例判 False（ADR-0002）：修正 markdown 的 ✅ bug 与 compare 的 None；顺带修 engine 连通用例无回复也算通过的 bug - pass_rate 口径改为用例级（CONTEXT.md 词条），规则级保留在 passed_rules/total_rules；CLI 对比标签同步更正 - 修 RunRepository.update 漏拷 scenario_version/triggered_by 的字段漂移
69 lines
2.3 KiB
Python
69 lines
2.3 KiB
Python
"""CLI commands for report generation."""
|
|
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
import typer
|
|
from agenteval.evaluation.report import render_html_report, render_json_report, save_report
|
|
from agenteval.storage.repository import RunRepository
|
|
from rich.console import Console
|
|
|
|
app = typer.Typer(help="评测报告")
|
|
console = Console()
|
|
|
|
|
|
@app.command("show", help="查看报告")
|
|
def show_report(
|
|
run_id: str,
|
|
fmt: str = typer.Option("json", "--format", help="输出格式: json | html"),
|
|
) -> None:
|
|
run = RunRepository().get(run_id)
|
|
if not run:
|
|
console.print(f"评测记录不存在: {run_id}", style="red")
|
|
raise typer.Exit(1)
|
|
|
|
if fmt == "json":
|
|
console.print(render_json_report(run_id))
|
|
elif fmt == "html":
|
|
console.print(render_html_report(run_id))
|
|
else:
|
|
console.print(f"不支持的格式: {fmt}", style="red")
|
|
raise typer.Exit(1)
|
|
|
|
|
|
@app.command("generate", help="生成报告并保存到文件")
|
|
def generate_report(
|
|
run_id: str,
|
|
fmt: str = typer.Option("html", "--format", help="输出格式: json | html"),
|
|
output_dir: Optional[Path] = typer.Option(None, help="输出目录"),
|
|
) -> None:
|
|
run = RunRepository().get(run_id)
|
|
if not run:
|
|
console.print(f"评测记录不存在: {run_id}", style="red")
|
|
raise typer.Exit(1)
|
|
|
|
path = save_report(run_id, fmt=fmt, output_dir=output_dir)
|
|
console.print(f"报告已保存: {path}")
|
|
|
|
|
|
@app.command("compare", help="对比两次评测报告")
|
|
def compare_reports(
|
|
run_id_1: str,
|
|
run_id_2: str,
|
|
) -> None:
|
|
from agenteval.evaluation.report import generate_report
|
|
|
|
r1 = RunRepository().get(run_id_1)
|
|
r2 = RunRepository().get(run_id_2)
|
|
if not r1 or not r2:
|
|
console.print("评测记录不存在", style="red")
|
|
raise typer.Exit(1)
|
|
|
|
report1 = generate_report(run_id_1)
|
|
report2 = generate_report(run_id_2)
|
|
|
|
console.print(f"对比: {run_id_1} vs {run_id_2}")
|
|
console.print(f" 用例数: {report1['summary']['total_cases']} -> {report2['summary']['total_cases']}")
|
|
console.print(f" 通过用例: {report1['summary']['passed_cases']} -> {report2['summary']['passed_cases']}")
|
|
console.print(f" 通过率: {report1['summary']['pass_rate']:.2%} -> {report2['summary']['pass_rate']:.2%}")
|