fix(engine): dynamic 生成失败原因持久化到 run.summary.case_errors
## 背景 用户反馈动态问诊评测「执行不下去」。诊断发现：dynamic 用例的 LLM 消息 生成 API 调用失败（0.2s 瞬间 failed，凭证/参数问题），引擎正确地标记 case 失败——但失败的具体原因（如 401 详情）只 emit 到 WebSocket，从不 写入 run.summary。导致 run 记录只有 total_rules:0 failed，DB/报告查不到 任何原因，用户和排查者都无从下手。 ## 修复 - EvalEngine 新增 self._case_errors 收集致命的 case 级错误 - _generate_messages 的 8 个失败点统一走 _fail() helper：既 emit 到 WebSocket，也记录到 _case_errors（含 case_id + stage + 具体 error） - run() 汇总时把 _case_errors 写入 summary["case_errors"] - 新增测试：dynamic 生成失败时 summary.case_errors 必须含原因（补上 之前 KNOWN-2 记录的 _generate_messages 测试盲区） ## 注 这不是导致失败的 bug（失败源于外部 API 凭证/参数），而是让失败「可诊断」 的可用性修复。用户需自查 llm_config 的 api_key 是否有效/model 是否被 该端点接受。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sinohqb
parent
c4962ddadf
commit
867d4e3ff1
@ -83,6 +83,10 @@ class EvalEngine:
|
||||
self.cancel_token = cancel_token or asyncio.Event()
|
||||
self.timeout_config = timeout_config or TimeoutConfig()
|
||||
self._case_semaphore = asyncio.Semaphore(max(1, max_concurrent_cases))
|
||||
# Collects fatal case-level errors (e.g. dynamic message generation
|
||||
# failures) so their cause is persisted into run.summary — not just
|
||||
# emitted transiently over WebSocket.
|
||||
self._case_errors: list[dict[str, str]] = []
|
||||
|
||||
# ── public entry point ────────────────────────────────────────────
|
||||
|
||||
@ -162,6 +166,10 @@ class EvalEngine:
|
||||
"passed_rules": passed_rules,
|
||||
"pass_rate": round(passed_rules / total_rules, 4) if total_rules else 0.0,
|
||||
}
|
||||
# Surface fatal case-level errors (e.g. dynamic generation failures)
|
||||
# so the report / DB record shows *why* a run produced no results.
|
||||
if self._case_errors:
|
||||
summary["case_errors"] = self._case_errors
|
||||
run.status = RunStatus.COMPLETED
|
||||
run.completed_at = utc_now()
|
||||
run.summary = summary
|
||||
@ -456,24 +464,24 @@ class EvalEngine:
|
||||
progress_callback: Optional[ProgressCallback],
|
||||
) -> list[str]:
|
||||
"""Use LLM to generate test messages for dynamic cases."""
|
||||
|
||||
async def _fail(msg: str) -> list[str]:
|
||||
# Persist the reason into run-level case_errors (surfaced in summary),
|
||||
# not just a transient WebSocket emit that's lost after the run.
|
||||
self._case_errors.append({"case_id": case.id, "stage": "generate_messages", "error": msg})
|
||||
await self._emit(progress_callback, "error", {"error": msg, "case_id": case.id})
|
||||
return []
|
||||
|
||||
llm_config = self.scenario.llm_config
|
||||
if not llm_config:
|
||||
await self._emit(
|
||||
progress_callback,
|
||||
"error",
|
||||
{
|
||||
"error": "动态用例需要配置 llm_config",
|
||||
},
|
||||
)
|
||||
return []
|
||||
return await _fail("动态用例需要配置 llm_config")
|
||||
|
||||
api_url = llm_config.get("api_url")
|
||||
api_key = llm_config.get("api_key")
|
||||
model = llm_config.get("model", "doubao-seed-2.0-lite")
|
||||
|
||||
if not api_url:
|
||||
await self._emit(progress_callback, "error", {"error": "llm_config 缺少 api_url"})
|
||||
return []
|
||||
return await _fail("llm_config 缺少 api_url")
|
||||
|
||||
turns = case.turns or 3
|
||||
prompt = case.prompt or "请生成一些测试问题"
|
||||
@ -504,35 +512,19 @@ class EvalEngine:
|
||||
resp.raise_for_status()
|
||||
content = extract_content_from_llm_response(resp.json())
|
||||
if not content:
|
||||
await self._emit(
|
||||
progress_callback,
|
||||
"error",
|
||||
{
|
||||
"error": "LLM 返回内容为空或无法解析",
|
||||
},
|
||||
)
|
||||
return []
|
||||
return await _fail("LLM 返回内容为空或无法解析")
|
||||
|
||||
try:
|
||||
parsed = parse_json_from_llm_text(content)
|
||||
except (ValueError, Exception) as parse_exc:
|
||||
await self._emit(
|
||||
progress_callback,
|
||||
"error",
|
||||
{
|
||||
"error": f"LLM 返回无法解析为数组: {parse_exc}",
|
||||
},
|
||||
)
|
||||
return []
|
||||
return await _fail(f"LLM 返回无法解析为数组: {parse_exc}")
|
||||
|
||||
if not isinstance(parsed, list):
|
||||
await self._emit(progress_callback, "error", {"error": "LLM 返回的不是数组"})
|
||||
return []
|
||||
return await _fail("LLM 返回的不是数组")
|
||||
|
||||
messages = [str(m) for m in parsed if isinstance(m, str) and m.strip()]
|
||||
if not messages:
|
||||
await self._emit(progress_callback, "error", {"error": "LLM 返回的消息为空"})
|
||||
return []
|
||||
return await _fail("LLM 返回的消息为空")
|
||||
|
||||
await self._emit(
|
||||
progress_callback,
|
||||
@ -545,8 +537,7 @@ class EvalEngine:
|
||||
return messages
|
||||
|
||||
except Exception as exc:
|
||||
await self._emit(progress_callback, "error", {"error": f"LLM 生成消息失败: {exc}"})
|
||||
return []
|
||||
return await _fail(f"LLM 生成消息失败: {exc}")
|
||||
|
||||
# ── helpers ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@ -304,3 +304,31 @@ async def test_send_failure_aborts_case(db_session):
|
||||
assert run.status == RunStatus.COMPLETED
|
||||
assert run.summary["failed_cases"] == 1
|
||||
assert run.summary["passed_cases"] == 1
|
||||
|
||||
|
||||
# ── dynamic case generation failure ──────────────────────────────────────
|
||||
|
||||
async def test_dynamic_generation_failure_records_case_error(db_session):
|
||||
"""A dynamic case whose message generation fails must persist the reason
|
||||
into run.summary.case_errors — not just emit it transiently. Otherwise a
|
||||
run shows 0 rules / failed with no discoverable cause."""
|
||||
scenario = Scenario(
|
||||
id="s-1", name="dynamic",
|
||||
cases=[Case(id="dyn-1", type=CaseType.DYNAMIC, prompt="生成问题", turns=3)],
|
||||
llm_config=None, # 缺 llm_config → 生成消息立即失败
|
||||
)
|
||||
channel = MockChannel()
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
|
||||
run = await engine.run()
|
||||
|
||||
assert run.status == RunStatus.COMPLETED
|
||||
assert run.summary["failed_cases"] == 1
|
||||
assert run.summary["total_rules"] == 0
|
||||
# 关键：失败原因被持久化到 summary，可在报告 / DB 查看
|
||||
assert "case_errors" in run.summary
|
||||
assert run.summary["case_errors"][0]["case_id"] == "dyn-1"
|
||||
assert "llm_config" in run.summary["case_errors"][0]["error"]
|
||||
# 被测通道不应被调用（生成阶段就失败了）
|
||||
assert channel.send_calls == 0
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user