12481cd1b8
## 核心变更
### 规则层全面异步化（DEBT-1）
- EvalRule.evaluate() 签名改为 async def，全量同步改造（无兼容层）
- LlmScoreRule._call_llm: requests.post → httpx.AsyncClient，彻底消除事件循环阻塞
- engine._save_rule_results: rule.evaluate() → await rule.evaluate()
### 工具函数去重（DEBT-2）
- 新建 agenteval/utils/llm.py，统一三个函数：
- extract_reply_text (原 5 处重复)
- extract_content_from_llm_response (原 2 处重复)
- parse_json_from_llm_text (统一 LLM 输出 JSON 解析)
- engine.py / llm_score.py / runs.py / report.py 全部切换到 utils.llm
### HTTP 通用通道（S1-3）
- 新建 channels/http.py (HttpChannel)
- 配置化 send_url / reply_url 模板 ({message}, {msg_id} 占位)
- dot-path 提取 msg_id 和 reply_text
- 可选 reply_ready_path 就绪标志
- 长连接 AsyncClient 复用
- ChannelFactory 注册 ChannelType.HTTP → HttpChannel
### 测试
- 新增 tests/unit/test_http_channel_and_rules.py (19 个测试)
- _get_path / health_check / send / poll_reply / 超时 / 就绪标志 / async 规则评估
- 测试总数：24 → 43，全部通过
Co-Authored-By: Claude <noreply@anthropic.com>
201 lines
7.7 KiB
Python
201 lines
7.7 KiB
Python
"""Report generation for evaluation runs."""
|
||
|
||
import json
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any, Optional
|
||
|
||
from jinja2 import Template
|
||
|
||
from agenteval.storage.db import DATA_DIR
|
||
from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository
|
||
from agenteval.utils.llm import extract_reply_text
|
||
|
||
HTML_TEMPLATE = """<!DOCTYPE html>
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<title>评测报告 - {{ report.run_id }}</title>
|
||
<style>
|
||
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; margin: 40px; background: #f5f5f5; }
|
||
.container { max-width: 960px; margin: 0 auto; background: #fff; padding: 32px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.05); }
|
||
h1 { margin-top: 0; }
|
||
.summary { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 16px; margin: 24px 0; }
|
||
.card { background: #fafafa; border-radius: 6px; padding: 16px; text-align: center; }
|
||
.card .value { font-size: 24px; font-weight: 700; }
|
||
.card .label { color: #666; font-size: 14px; margin-top: 4px; }
|
||
.case { border: 1px solid #e0e0e0; border-radius: 6px; margin: 16px 0; padding: 16px; }
|
||
.case-title { font-weight: 600; margin-bottom: 8px; }
|
||
.turn { background: #f9f9f9; border-radius: 4px; padding: 12px; margin: 8px 0; }
|
||
.message-label { color: #666; font-size: 12px; }
|
||
.rule { display: flex; align-items: center; gap: 8px; margin: 6px 0; }
|
||
.badge { padding: 2px 8px; border-radius: 4px; font-size: 12px; }
|
||
.pass { background: #e6f7e6; color: #2e7d32; }
|
||
.fail { background: #ffebee; color: #c62828; }
|
||
pre { white-space: pre-wrap; word-break: break-word; background: #f5f5f5; padding: 8px; border-radius: 4px; }
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<div class="container">
|
||
<h1>评测报告</h1>
|
||
<p>评测对象：{{ report.target_name }}（{{ report.target_id }}）</p>
|
||
<p>评测场景：{{ report.scenario_name }}（{{ report.scenario_id }}）</p>
|
||
<p>执行时间：{{ report.started_at }} 至 {{ report.completed_at or '进行中' }}</p>
|
||
|
||
<div class="summary">
|
||
<div class="card">
|
||
<div class="value">{{ report.summary.total_cases }}</div>
|
||
<div class="label">用例总数</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="value">{{ report.summary.passed_cases }}</div>
|
||
<div class="label">通过用例</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="value">{{ report.summary.total_rules }}</div>
|
||
<div class="label">规则总数</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="value">{{ "%.2f"|format(report.summary.pass_rate * 100) }}%</div>
|
||
<div class="label">规则通过率</div>
|
||
</div>
|
||
</div>
|
||
|
||
{% for case in report.cases %}
|
||
<div class="case">
|
||
<div class="case-title">用例 {{ case.case_id }}</div>
|
||
{% for turn in case.turns %}
|
||
<div class="turn">
|
||
<div class="message-label">用户消息</div>
|
||
<pre>{{ turn.sent_text }}</pre>
|
||
<div class="message-label">智能体回复（{{ turn.latency_ms }}ms）</div>
|
||
<pre>{{ turn.reply_text or '（无回复）' }}</pre>
|
||
</div>
|
||
{% endfor %}
|
||
<div>
|
||
{% for result in case.results %}
|
||
<div class="rule">
|
||
<span class="badge {{ 'pass' if result.passed else 'fail' }}">{{ '通过' if result.passed else '失败' }}</span>
|
||
<span>{{ result.rule_type }}: {{ result.reason }}</span>
|
||
</div>
|
||
{% endfor %}
|
||
</div>
|
||
</div>
|
||
{% endfor %}
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
|
||
def _extract_text(data: Any) -> str:
|
||
return extract_reply_text(data)
|
||
|
||
|
||
def generate_report(run_id: str, session=None) -> dict[str, Any]:
|
||
"""Build a structured report dict for a run."""
|
||
run_repo = RunRepository(session)
|
||
target_repo = TargetRepository(session)
|
||
scenario_repo = ScenarioRepository(session)
|
||
|
||
run = run_repo.get(run_id)
|
||
if not run:
|
||
raise ValueError(f"run not found: {run_id}")
|
||
|
||
target = target_repo.get(run.target_id)
|
||
scenario = scenario_repo.get(run.scenario_id)
|
||
|
||
turns = run_repo.get_turns(run_id)
|
||
results = run_repo.get_results(run_id)
|
||
|
||
# Group by case
|
||
case_map: dict[str, dict[str, Any]] = {}
|
||
for turn in turns:
|
||
case_map.setdefault(turn.case_id, {"turns": [], "results": []})
|
||
sent = turn.get_sent_message()
|
||
reply = turn.get_reply()
|
||
case_map[turn.case_id]["turns"].append(
|
||
{
|
||
"round": turn.round_index,
|
||
"sent_text": _extract_text(sent.get("msgBody")),
|
||
"reply_text": _extract_text(reply.get("msgBody") if reply else None),
|
||
"latency_ms": turn.latency_ms,
|
||
"question_msg_id": turn.question_msg_id,
|
||
}
|
||
)
|
||
|
||
for result in results:
|
||
case_map.setdefault(result.case_id, {"turns": [], "results": []})
|
||
case_map[result.case_id]["results"].append(
|
||
{
|
||
"rule_type": result.rule_type,
|
||
"passed": result.passed,
|
||
"score": result.score,
|
||
"reason": result.reason,
|
||
}
|
||
)
|
||
|
||
cases = []
|
||
for case_id in sorted(case_map.keys()):
|
||
item = case_map[case_id]
|
||
cases.append(
|
||
{
|
||
"case_id": case_id,
|
||
"turns": sorted(item["turns"], key=lambda x: x["round"]),
|
||
"results": item["results"],
|
||
}
|
||
)
|
||
|
||
summary = run.summary or {}
|
||
return {
|
||
"run_id": run.id,
|
||
"target_id": run.target_id,
|
||
"target_name": target.name if target else "未知",
|
||
"scenario_id": run.scenario_id,
|
||
"scenario_name": scenario.name if scenario else "未知",
|
||
"status": run.status.value,
|
||
"started_at": run.started_at.isoformat() if run.started_at else None,
|
||
"completed_at": run.completed_at.isoformat() if run.completed_at else None,
|
||
"summary": {
|
||
"total_cases": summary.get("total_cases", 0),
|
||
"passed_cases": summary.get("passed_cases", 0),
|
||
"failed_cases": summary.get("failed_cases", 0),
|
||
"total_rules": summary.get("total_rules", 0),
|
||
"passed_rules": summary.get("passed_rules", 0),
|
||
"pass_rate": summary.get("pass_rate", 0.0),
|
||
},
|
||
"cases": cases,
|
||
}
|
||
|
||
|
||
def render_json_report(run_id: str, session=None) -> str:
|
||
"""Render a report as JSON string."""
|
||
report = generate_report(run_id, session)
|
||
return json.dumps(report, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def render_html_report(run_id: str, session=None) -> str:
|
||
"""Render a report as HTML string."""
|
||
report = generate_report(run_id, session)
|
||
template = Template(HTML_TEMPLATE)
|
||
return template.render(report=report)
|
||
|
||
|
||
def save_report(run_id: str, fmt: str = "html", output_dir: Optional[Path] = None) -> Path:
|
||
"""Generate and save a report to disk."""
|
||
output_dir = output_dir or DATA_DIR / "reports"
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
if fmt == "html":
|
||
content = render_html_report(run_id)
|
||
path = output_dir / f"report_{run_id}_{timestamp}.html"
|
||
elif fmt == "json":
|
||
content = render_json_report(run_id)
|
||
path = output_dir / f"report_{run_id}_{timestamp}.json"
|
||
else:
|
||
raise ValueError(f"unsupported report format: {fmt}")
|
||
|
||
path.write_text(content, encoding="utf-8")
|
||
return path
|