a77cd83e6a
## 新增功能 - 文件管理模块：分类树 + 文件上传/下载/删除 - 文件上传支持拖拽（Dragger）+ 手动上传（customRequest 模式） ## 页面布局统一（参照评测执行页） - 仪表盘/评测对象/评测场景/评测报告 全部改为全高 flex 布局 - 统一内联页头样式（h2 + 竖线分隔 + 描述） - 表格撑满高度、overflow 处理 - 每页添加刷新按钮 ## Bug 修复 - 分类树操作按钮 hover 不可见（CSS 规则缺失） - 文件上传失败（multipart boundary 缺失） - LLM API 响应 content blocks 数组格式支持（_extract_content_from_api_response） - response_time_max_ms 被静默忽略（隐式规则传空 params） - 空 messages 导致 IndexError 崩溃 - poll_reply 异常中止整个 run（缺 try/catch） - engine finally 未关闭 session - 3 个页面 UTC 时间戳解析偏差 8 小时 ## 后端 - EvalEngine: poll_reply 异常保护、空 dialog 保护、session 关闭 - LLM API 响应解析支持 content-block-array 格式 - 隐式 response_time 规则正确传递 max_ms 参数 ## 前端 - api.ts: 移除手动 Content-Type（让浏览器自动添加 boundary） - Files.tsx: customRequest 替代 beforeUpload、布局优化 - index.css: 分类树 hover 规则 - Targets/Scenarios/Home/Reports: 全高布局改造 - 3 个页面时间戳改用 formatDateTime()（修复 UTC 偏差） Co-Authored-By: Claude <noreply@anthropic.com>
473 lines
18 KiB
Python
473 lines
18 KiB
Python
"""Evaluation execution engine.
|
|
|
|
Async-first implementation: channels and LLM calls are awaited cooperatively,
|
|
so multiple cases can run concurrently and a run can be cancelled mid-flight
|
|
via an ``asyncio.Event`` cancel token.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import uuid
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Awaitable, Callable, Optional
|
|
|
|
import httpx
|
|
|
|
from agenteval.channels.base import EvalChannel
|
|
from agenteval.channels.factory import ChannelFactory
|
|
from agenteval.evaluation.rules import get_rule
|
|
from agenteval.models import Case, CaseType, EvalResult, EvalRun, EvalTarget, RunStatus, Scenario, Turn
|
|
from agenteval.storage.db import get_session, utc_now
|
|
from agenteval.storage.repository import ResultRepository, RunRepository
|
|
|
|
|
|
# Progress callbacks may be sync or async; the engine awaits the result if
|
|
# it is a coroutine, otherwise treats it as a plain function.
|
|
ProgressCallback = Callable[[str, dict[str, Any]], Any]
|
|
|
|
|
|
class CancelledError(RuntimeError):
|
|
"""Raised inside the engine when the cancel token fires."""
|
|
|
|
|
|
@dataclass
|
|
class TimeoutConfig:
|
|
"""Per-operation timeouts (all in seconds)."""
|
|
|
|
poll_reply: float = 30.0
|
|
llm_generate: float = 60.0
|
|
|
|
|
|
def _build_send_message(content: str) -> dict[str, Any]:
|
|
return {
|
|
"msgType": "text",
|
|
"msgBody": {"content": content},
|
|
}
|
|
|
|
|
|
def _extract_content_from_api_response(data: dict) -> str:
|
|
"""Extract text content from an LLM API response.
|
|
|
|
Handles both OpenAI format (choices[0].message.content as string)
|
|
and content-block-array format used by Anthropic-compatible APIs
|
|
(choices[0].message.content as list of {type, text/text} blocks).
|
|
"""
|
|
try:
|
|
content = data["choices"][0]["message"]["content"]
|
|
except (KeyError, IndexError, TypeError):
|
|
return ""
|
|
if isinstance(content, str):
|
|
return content
|
|
if isinstance(content, list):
|
|
parts = []
|
|
for block in content:
|
|
if not isinstance(block, dict):
|
|
continue
|
|
# Anthropic format: {"type": "text", "text": "..."}
|
|
# Some APIs may use "content" key instead of "text"
|
|
if block.get("type") == "text":
|
|
parts.append(block.get("text") or block.get("content") or "")
|
|
else:
|
|
# Non-text blocks (tool_use, image, etc.) — skip
|
|
pass
|
|
return "\n".join(parts)
|
|
return str(content)
|
|
|
|
|
|
def _extract_reply_text(reply: Any) -> str:
|
|
if reply is None:
|
|
return ""
|
|
if isinstance(reply, str):
|
|
return reply
|
|
if isinstance(reply, dict):
|
|
body = reply.get("msgBody") or reply.get("content", "")
|
|
if isinstance(body, dict):
|
|
return body.get("content", "")
|
|
return str(body)
|
|
return str(reply)
|
|
|
|
|
|
class EvalEngine:
|
|
"""Execute evaluation scenarios against targets.
|
|
|
|
The engine is async so the (slow, network-bound) channel and LLM calls
|
|
can be awaited cooperatively. Database writes remain synchronous for now
|
|
(SQLite + StaticPool); they are fast enough not to block the event loop
|
|
in practice.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
target: EvalTarget,
|
|
scenario: Scenario,
|
|
session=None,
|
|
run_repo: Optional[RunRepository] = None,
|
|
result_repo: Optional[ResultRepository] = None,
|
|
cancel_token: Optional[asyncio.Event] = None,
|
|
timeout_config: Optional[TimeoutConfig] = None,
|
|
max_concurrent_cases: int = 1,
|
|
):
|
|
self.target = target
|
|
self.scenario = scenario
|
|
self.channel: EvalChannel = ChannelFactory.create(target)
|
|
self.session = session or get_session()
|
|
self.run_repo = run_repo or RunRepository(self.session)
|
|
self.result_repo = result_repo or ResultRepository(self.session)
|
|
self.cancel_token = cancel_token or asyncio.Event()
|
|
self.timeout_config = timeout_config or TimeoutConfig()
|
|
self._case_semaphore = asyncio.Semaphore(max(1, max_concurrent_cases))
|
|
|
|
# ── public entry point ────────────────────────────────────────────
|
|
|
|
async def run(
|
|
self,
|
|
progress_callback: Optional[ProgressCallback] = None,
|
|
existing_run: Optional[EvalRun] = None,
|
|
) -> EvalRun:
|
|
"""Run the evaluation and return the completed run record.
|
|
|
|
Cancellation is cooperative: set ``cancel_token`` and the engine will
|
|
mark the run as FAILED with ``cancelled_by_user`` at the next checkpoint.
|
|
"""
|
|
if existing_run:
|
|
run = existing_run
|
|
run.status = RunStatus.RUNNING
|
|
run.started_at = utc_now()
|
|
run = self.run_repo.update(run) or run
|
|
else:
|
|
run = EvalRun(
|
|
id=str(uuid.uuid4()),
|
|
target_id=self.target.id or "",
|
|
scenario_id=self.scenario.id or "",
|
|
status=RunStatus.RUNNING,
|
|
started_at=utc_now(),
|
|
)
|
|
run = self.run_repo.create(run)
|
|
|
|
try:
|
|
total_cases = len(self.scenario.cases)
|
|
passed_cases = 0
|
|
failed_cases = 0
|
|
|
|
for idx, case in enumerate(self.scenario.cases, start=1):
|
|
self._check_cancel()
|
|
await self._emit(progress_callback, "case_start", {
|
|
"index": idx, "total": total_cases, "case_id": case.id,
|
|
})
|
|
async with self._case_semaphore:
|
|
case_passed, rule_pass, rule_total = await self._run_case(
|
|
run, case, progress_callback,
|
|
)
|
|
if case_passed:
|
|
passed_cases += 1
|
|
else:
|
|
failed_cases += 1
|
|
await self._emit(progress_callback, "case_end", {
|
|
"index": idx, "total": total_cases, "case_id": case.id,
|
|
"passed": case_passed,
|
|
"rule_pass_count": rule_pass, "rule_total": rule_total,
|
|
})
|
|
|
|
results = self.run_repo.get_results(run.id)
|
|
total_rules = len(results)
|
|
passed_rules = sum(1 for r in results if r.passed)
|
|
|
|
summary = {
|
|
"total_cases": total_cases,
|
|
"passed_cases": passed_cases,
|
|
"failed_cases": failed_cases,
|
|
"total_rules": total_rules,
|
|
"passed_rules": passed_rules,
|
|
"pass_rate": round(passed_rules / total_rules, 4) if total_rules else 0.0,
|
|
}
|
|
run.status = RunStatus.COMPLETED
|
|
run.completed_at = utc_now()
|
|
run.summary = summary
|
|
await self._emit(progress_callback, "run_completed", {
|
|
"status": "completed", "summary": summary,
|
|
})
|
|
except CancelledError:
|
|
run.status = RunStatus.FAILED
|
|
run.completed_at = utc_now()
|
|
run.summary = {
|
|
"error": {"code": "cancelled_by_user", "message": "评测已手动停止"},
|
|
}
|
|
await self._emit(progress_callback, "run_completed", {
|
|
"status": "failed", "reason": "cancelled",
|
|
"error": {"code": "cancelled_by_user", "message": "评测已手动停止"},
|
|
})
|
|
except Exception as exc:
|
|
run.status = RunStatus.FAILED
|
|
run.completed_at = utc_now()
|
|
run.summary = {"error": str(exc)}
|
|
await self._emit(progress_callback, "error", {"error": str(exc)})
|
|
await self._emit(progress_callback, "run_completed", {
|
|
"status": "failed", "error": str(exc),
|
|
})
|
|
raise
|
|
finally:
|
|
run = self.run_repo.update(run) or run
|
|
# Best-effort cleanup of the channel's HTTP client.
|
|
close = getattr(self.channel, "close", None)
|
|
if callable(close):
|
|
try:
|
|
result = close()
|
|
if asyncio.iscoroutine(result):
|
|
await result
|
|
except Exception:
|
|
pass
|
|
try:
|
|
self.session.close()
|
|
except Exception:
|
|
pass
|
|
|
|
return run
|
|
|
|
# ── case / turn execution ─────────────────────────────────────────
|
|
|
|
async def _run_case(
|
|
self,
|
|
run: EvalRun,
|
|
case: Case,
|
|
progress_callback: Optional[ProgressCallback],
|
|
) -> tuple[bool, int, int]:
|
|
"""Run a single case; returns (all_rules_passed, passed_rules, total_rules)."""
|
|
if case.type == CaseType.DYNAMIC:
|
|
generated = await self._generate_messages(case, progress_callback)
|
|
if not generated:
|
|
await self._emit(progress_callback, "error", {
|
|
"error": "LLM 未能生成测试消息", "case_id": case.id,
|
|
})
|
|
return False, 0, 0
|
|
case = case.model_copy(update={"messages": generated})
|
|
|
|
dialog: list[Turn] = []
|
|
|
|
for round_index, message in enumerate(case.messages, start=1):
|
|
self._check_cancel()
|
|
await self._emit(progress_callback, "turn_start", {
|
|
"run_id": run.id, "case_id": case.id,
|
|
"round": round_index, "message": message,
|
|
})
|
|
|
|
sent_at = utc_now()
|
|
send_result = await self.channel.send(message)
|
|
if not send_result.ok:
|
|
turn = Turn(
|
|
id=str(uuid.uuid4()),
|
|
run_id=run.id, case_id=case.id, round_index=round_index,
|
|
sent_message=_build_send_message(message), sent_at=sent_at,
|
|
)
|
|
self.result_repo.save_turn(turn)
|
|
await self._save_rule_results(run, case, turn, [], progress_callback)
|
|
await self._emit(progress_callback, "turn_error", {
|
|
"case_id": case.id, "round": round_index, "error": send_result.error,
|
|
})
|
|
return False, 0, 0
|
|
|
|
try:
|
|
reply = await self.channel.poll_reply(
|
|
send_result.question_msg_id or "",
|
|
timeout=self.timeout_config.poll_reply,
|
|
)
|
|
except Exception as poll_exc:
|
|
received_at = utc_now()
|
|
turn = Turn(
|
|
id=str(uuid.uuid4()),
|
|
run_id=run.id, case_id=case.id, round_index=round_index,
|
|
sent_message=_build_send_message(message), sent_at=sent_at,
|
|
question_msg_id=send_result.question_msg_id,
|
|
received_at=received_at,
|
|
)
|
|
self.result_repo.save_turn(turn)
|
|
await self._emit(progress_callback, "turn_error", {
|
|
"case_id": case.id, "round": round_index,
|
|
"error": f"poll_reply 异常: {poll_exc}",
|
|
})
|
|
return False, 0, 0
|
|
|
|
received_at = utc_now()
|
|
latency_ms = None
|
|
if sent_at and received_at:
|
|
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
|
|
|
|
turn = Turn(
|
|
id=str(uuid.uuid4()),
|
|
run_id=run.id, case_id=case.id, round_index=round_index,
|
|
sent_message=_build_send_message(message), sent_at=sent_at,
|
|
question_msg_id=send_result.question_msg_id,
|
|
reply=reply.raw_message if reply else None,
|
|
received_at=received_at, latency_ms=latency_ms,
|
|
)
|
|
self.result_repo.save_turn(turn)
|
|
dialog.append(turn)
|
|
|
|
await self._emit(progress_callback, "turn_end", {
|
|
"run_id": run.id, "case_id": case.id, "round": round_index,
|
|
"latency_ms": latency_ms,
|
|
"reply_text": _extract_reply_text(reply.raw_message if reply else None),
|
|
})
|
|
|
|
if not dialog:
|
|
return False, 0, 0
|
|
|
|
return await self._save_rule_results(run, case, dialog[-1], dialog, progress_callback)
|
|
|
|
async def _save_rule_results(
|
|
self,
|
|
run: EvalRun,
|
|
case: Case,
|
|
turn: Turn,
|
|
dialog: list[Turn],
|
|
progress_callback: Optional[ProgressCallback],
|
|
) -> tuple[bool, int, int]:
|
|
"""Apply rules and save results; returns (all_passed, passed_count, total_count)."""
|
|
from agenteval.models import EvalRuleConfig
|
|
|
|
rules_config: list[EvalRuleConfig] = list(case.eval_rules)
|
|
|
|
# If no explicit rules, derive implicit rules from expectations.
|
|
if not rules_config:
|
|
if case.expectations.response_time_max_ms:
|
|
rules_config.append(EvalRuleConfig(type="response_time", params={
|
|
"max_ms": case.expectations.response_time_max_ms,
|
|
}))
|
|
if case.expectations.keywords_include or case.expectations.keywords_exclude:
|
|
rules_config.append(
|
|
EvalRuleConfig(
|
|
type="keyword_match",
|
|
params={
|
|
"keywords": case.expectations.keywords_include,
|
|
"exclude_keywords": case.expectations.keywords_exclude,
|
|
},
|
|
)
|
|
)
|
|
|
|
all_passed = True
|
|
passed_count = 0
|
|
total_count = 0
|
|
for rule_config in rules_config:
|
|
rule = get_rule(rule_config.type, rule_config.params)
|
|
result = rule.evaluate(case, dialog)
|
|
eval_result = EvalResult(
|
|
id=str(uuid.uuid4()),
|
|
run_id=run.id, case_id=case.id, turn_id=turn.id or "",
|
|
rule_type=rule_config.type,
|
|
passed=result.passed, score=result.score, reason=result.reason,
|
|
)
|
|
self.result_repo.save_result(eval_result)
|
|
total_count += 1
|
|
if result.passed:
|
|
passed_count += 1
|
|
else:
|
|
all_passed = False
|
|
await self._emit(progress_callback, "rule_result", {
|
|
"run_id": run.id, "case_id": case.id,
|
|
"rule_type": rule_config.type,
|
|
"passed": result.passed, "score": result.score, "reason": result.reason,
|
|
})
|
|
|
|
return all_passed, passed_count, total_count
|
|
|
|
async def _generate_messages(
|
|
self, case: Case, progress_callback: Optional[ProgressCallback],
|
|
) -> list[str]:
|
|
"""Use LLM to generate test messages for dynamic cases."""
|
|
llm_config = self.scenario.llm_config
|
|
if not llm_config:
|
|
await self._emit(progress_callback, "error", {
|
|
"error": "动态用例需要配置 llm_config",
|
|
})
|
|
return []
|
|
|
|
api_url = llm_config.get("api_url")
|
|
api_key = llm_config.get("api_key")
|
|
model = llm_config.get("model", "doubao-seed-2.0-lite")
|
|
|
|
if not api_url:
|
|
await self._emit(progress_callback, "error", {"error": "llm_config 缺少 api_url"})
|
|
return []
|
|
|
|
turns = case.turns or 3
|
|
prompt = case.prompt or "请生成一些测试问题"
|
|
|
|
system_prompt = (
|
|
f"你需要扮演一个真实的用户/患者，根据以下要求生成 {turns} 条独立的测试问题。\n\n"
|
|
f"要求：{prompt}\n\n"
|
|
"输出格式要求：只输出一个 JSON 数组，包含 " + str(turns) + " 个字符串，每个字符串是一条消息。"
|
|
"不要输出任何解释、markdown 或其他内容。"
|
|
)
|
|
|
|
headers = {"Content-Type": "application/json"}
|
|
if api_key:
|
|
headers["Authorization"] = f"Bearer {api_key}"
|
|
|
|
payload = {
|
|
"model": model,
|
|
"messages": [
|
|
{"role": "system", "content": system_prompt},
|
|
{"role": "user", "content": f"请生成 {turns} 条测试消息"},
|
|
],
|
|
"temperature": 0.7,
|
|
}
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=self.timeout_config.llm_generate) as client:
|
|
resp = await client.post(api_url, headers=headers, json=payload)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
content = _extract_content_from_api_response(data)
|
|
if not content:
|
|
await self._emit(progress_callback, "error", {
|
|
"error": "LLM 返回内容为空或无法解析",
|
|
})
|
|
return []
|
|
|
|
try:
|
|
parsed = json.loads(content)
|
|
except json.JSONDecodeError:
|
|
start = content.find("[")
|
|
end = content.rfind("]")
|
|
if start == -1 or end == -1:
|
|
await self._emit(progress_callback, "error", {
|
|
"error": f"LLM 返回无法解析为数组: {content[:200]}",
|
|
})
|
|
return []
|
|
parsed = json.loads(content[start:end + 1])
|
|
|
|
if not isinstance(parsed, list):
|
|
await self._emit(progress_callback, "error", {"error": "LLM 返回的不是数组"})
|
|
return []
|
|
|
|
messages = [str(m) for m in parsed if isinstance(m, str) and m.strip()]
|
|
if not messages:
|
|
await self._emit(progress_callback, "error", {"error": "LLM 返回的消息为空"})
|
|
return []
|
|
|
|
await self._emit(progress_callback, "messages_generated", {
|
|
"case_id": case.id, "messages": messages,
|
|
})
|
|
return messages
|
|
|
|
except Exception as exc:
|
|
await self._emit(progress_callback, "error", {"error": f"LLM 生成消息失败: {exc}"})
|
|
return []
|
|
|
|
# ── helpers ────────────────────────────────────────────────────────
|
|
|
|
def _check_cancel(self) -> None:
|
|
if self.cancel_token.is_set():
|
|
raise CancelledError("run cancelled")
|
|
|
|
async def _emit(
|
|
self, callback: Optional[ProgressCallback], event: str, data: dict[str, Any],
|
|
) -> None:
|
|
if not callback:
|
|
return
|
|
try:
|
|
result = callback(event, data)
|
|
if asyncio.iscoroutine(result):
|
|
await result
|
|
except Exception:
|
|
pass
|