From 867d4e3ff1ba62ed6bb7c1c9362ec5490738d110 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Fri, 17 Jul 2026 16:29:51 +0800 Subject: [PATCH] =?UTF-8?q?fix(engine):=20dynamic=20=E7=94=9F=E6=88=90?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E5=8E=9F=E5=9B=A0=E6=8C=81=E4=B9=85=E5=8C=96?= =?UTF-8?q?=E5=88=B0=20run.summary.case=5Ferrors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 背景 用户反馈动态问诊评测「执行不下去」。诊断发现：dynamic 用例的 LLM 消息 生成 API 调用失败（0.2s 瞬间 failed，凭证/参数问题），引擎正确地标记 case 失败——但失败的具体原因（如 401 详情）只 emit 到 WebSocket，从不 写入 run.summary。导致 run 记录只有 total_rules:0 failed，DB/报告查不到 任何原因，用户和排查者都无从下手。 ## 修复 - EvalEngine 新增 self._case_errors 收集致命的 case 级错误 - _generate_messages 的 8 个失败点统一走 _fail() helper：既 emit 到 WebSocket，也记录到 _case_errors（含 case_id + stage + 具体 error） - run() 汇总时把 _case_errors 写入 summary["case_errors"] - 新增测试：dynamic 生成失败时 summary.case_errors 必须含原因（补上 之前 KNOWN-2 记录的 _generate_messages 测试盲区） ## 注 这不是导致失败的 bug（失败源于外部 API 凭证/参数），而是让失败「可诊断」 的可用性修复。用户需自查 llm_config 的 api_key 是否有效/model 是否被 该端点接受。 Co-Authored-By: Claude --- backend/agenteval/evaluation/engine.py | 55 +++++++++++--------------- tests/unit/test_engine.py | 28 +++++++++++++ 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index 83693d3..82c6d93 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -83,6 +83,10 @@ class EvalEngine: self.cancel_token = cancel_token or asyncio.Event() self.timeout_config = timeout_config or TimeoutConfig() self._case_semaphore = asyncio.Semaphore(max(1, max_concurrent_cases)) + # Collects fatal case-level errors (e.g. dynamic message generation + # failures) so their cause is persisted into run.summary — not just + # emitted transiently over WebSocket. + self._case_errors: list[dict[str, str]] = [] # ── public entry point ──────────────────────────────────────────── @@ -162,6 +166,10 @@ class EvalEngine: "passed_rules": passed_rules, "pass_rate": round(passed_rules / total_rules, 4) if total_rules else 0.0, } + # Surface fatal case-level errors (e.g. dynamic generation failures) + # so the report / DB record shows *why* a run produced no results. + if self._case_errors: + summary["case_errors"] = self._case_errors run.status = RunStatus.COMPLETED run.completed_at = utc_now() run.summary = summary @@ -456,24 +464,24 @@ class EvalEngine: progress_callback: Optional[ProgressCallback], ) -> list[str]: """Use LLM to generate test messages for dynamic cases.""" + + async def _fail(msg: str) -> list[str]: + # Persist the reason into run-level case_errors (surfaced in summary), + # not just a transient WebSocket emit that's lost after the run. + self._case_errors.append({"case_id": case.id, "stage": "generate_messages", "error": msg}) + await self._emit(progress_callback, "error", {"error": msg, "case_id": case.id}) + return [] + llm_config = self.scenario.llm_config if not llm_config: - await self._emit( - progress_callback, - "error", - { - "error": "动态用例需要配置 llm_config", - }, - ) - return [] + return await _fail("动态用例需要配置 llm_config") api_url = llm_config.get("api_url") api_key = llm_config.get("api_key") model = llm_config.get("model", "doubao-seed-2.0-lite") if not api_url: - await self._emit(progress_callback, "error", {"error": "llm_config 缺少 api_url"}) - return [] + return await _fail("llm_config 缺少 api_url") turns = case.turns or 3 prompt = case.prompt or "请生成一些测试问题" @@ -504,35 +512,19 @@ class EvalEngine: resp.raise_for_status() content = extract_content_from_llm_response(resp.json()) if not content: - await self._emit( - progress_callback, - "error", - { - "error": "LLM 返回内容为空或无法解析", - }, - ) - return [] + return await _fail("LLM 返回内容为空或无法解析") try: parsed = parse_json_from_llm_text(content) except (ValueError, Exception) as parse_exc: - await self._emit( - progress_callback, - "error", - { - "error": f"LLM 返回无法解析为数组: {parse_exc}", - }, - ) - return [] + return await _fail(f"LLM 返回无法解析为数组: {parse_exc}") if not isinstance(parsed, list): - await self._emit(progress_callback, "error", {"error": "LLM 返回的不是数组"}) - return [] + return await _fail("LLM 返回的不是数组") messages = [str(m) for m in parsed if isinstance(m, str) and m.strip()] if not messages: - await self._emit(progress_callback, "error", {"error": "LLM 返回的消息为空"}) - return [] + return await _fail("LLM 返回的消息为空") await self._emit( progress_callback, @@ -545,8 +537,7 @@ class EvalEngine: return messages except Exception as exc: - await self._emit(progress_callback, "error", {"error": f"LLM 生成消息失败: {exc}"}) - return [] + return await _fail(f"LLM 生成消息失败: {exc}") # ── helpers ──────────────────────────────────────────────────────── diff --git a/tests/unit/test_engine.py b/tests/unit/test_engine.py index 3b2b287..7e00045 100644 --- a/tests/unit/test_engine.py +++ b/tests/unit/test_engine.py @@ -304,3 +304,31 @@ async def test_send_failure_aborts_case(db_session): assert run.status == RunStatus.COMPLETED assert run.summary["failed_cases"] == 1 assert run.summary["passed_cases"] == 1 + + +# ── dynamic case generation failure ────────────────────────────────────── + +async def test_dynamic_generation_failure_records_case_error(db_session): + """A dynamic case whose message generation fails must persist the reason + into run.summary.case_errors — not just emit it transiently. Otherwise a + run shows 0 rules / failed with no discoverable cause.""" + scenario = Scenario( + id="s-1", name="dynamic", + cases=[Case(id="dyn-1", type=CaseType.DYNAMIC, prompt="生成问题", turns=3)], + llm_config=None, # 缺 llm_config → 生成消息立即失败 + ) + channel = MockChannel() + engine = _build_engine(scenario, channel, session=db_session) + + run = await engine.run() + + assert run.status == RunStatus.COMPLETED + assert run.summary["failed_cases"] == 1 + assert run.summary["total_rules"] == 0 + # 关键：失败原因被持久化到 summary，可在报告 / DB 查看 + assert "case_errors" in run.summary + assert run.summary["case_errors"][0]["case_id"] == "dyn-1" + assert "llm_config" in run.summary["case_errors"][0]["error"] + # 被测通道不应被调用（生成阶段就失败了） + assert channel.send_calls == 0 +