From c4962ddadfed2b802d5b433eccbd34f7adb558c0 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Fri, 17 Jul 2026 16:15:07 +0800 Subject: [PATCH] =?UTF-8?q?fix(llm=5Fscore):=20=E4=BF=AE=E5=A4=8D=E5=A4=9A?= =?UTF-8?q?=E8=BD=AE=E7=94=A8=E4=BE=8B=20question=20=E6=8F=90=E5=8F=96?= =?UTF-8?q?=E9=94=99=E4=BD=8D=E5=AF=BC=E8=87=B4=E6=99=AE=E9=81=8D=E6=89=93?= =?UTF-8?q?=200=20=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 现象 多轮/动态用例的 llm_score 规则几乎全部返回 0.0/10 失败，即便被测对象 回复质量很高（实测 400+ 字的专业医疗回复也是 0 分）。 ## 根因 evaluate() 里 question 提取逻辑错误： if len(dialog) >= 2: question_text = extract_reply_text(dialog[-2].reply) # BUG dialog[-2].reply 是「上一轮智能体的回复」，被误当成「用户问题」。于是评分 LLM 收到的问答对是： - "用户问题" = 上一轮 AI 回复 - "智能体回复" = 当前轮 AI 回复 两段都是 AI 说的话、互不相关，评分 LLM 判定牛头不对马嘴 → 打 0 分。 单轮用例因 len(dialog)<2 走 sent_message 提取（正确），故不受影响； 问题只在多轮/dynamic 用例爆发。 ## 修复 - question 始终取当前轮 last_turn.sent_message（用户实际发送的问题）， 移除错误的 dialog[-2].reply 分支 - reason 增加评分 LLM 自己的理由（parsed["reason"]），便于未来诊断 - 新增回归测试：多轮场景验证 question 来自 last sent_message 而非 prev reply Co-Authored-By: Claude --- .../agenteval/evaluation/rules/llm_score.py | 11 +++-- tests/unit/test_llm_score.py | 47 +++++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/backend/agenteval/evaluation/rules/llm_score.py b/backend/agenteval/evaluation/rules/llm_score.py index 9193352..ec7b810 100644 --- a/backend/agenteval/evaluation/rules/llm_score.py +++ b/backend/agenteval/evaluation/rules/llm_score.py @@ -22,10 +22,11 @@ class LlmScoreRule(EvalRule): last_turn = dialog[-1] reply_text = extract_reply_text(last_turn.reply) + # 用户问题取自当前轮发送的消息（sent_message），而非上一轮的智能体回复。 + # 旧逻辑用 dialog[-2].reply 会把「上一轮 AI 回复」误当成「用户问题」， + # 导致多轮/动态用例里评分 LLM 收到牛头不对马嘴的问答对，普遍打 0 分。 question_text = "" - if len(dialog) >= 2: - question_text = extract_reply_text(dialog[-2].reply) or "" - if not question_text and last_turn.sent_message: + if last_turn.sent_message: body = last_turn.sent_message.get("msgBody", "") if isinstance(body, dict): question_text = body.get("content", "") @@ -49,10 +50,12 @@ class LlmScoreRule(EvalRule): return RuleResult(passed=False, reason=f"LLM 评分失败: {reason}") passed = score >= min_score + verdict = "通过" if passed else "未通过" + detail = f"；{reason}" if reason else "" return RuleResult( passed=passed, score=score / 10.0, - reason=f"LLM 评分 {score}/10，{'通过' if passed else '未通过'} (阈值 {min_score})", + reason=f"LLM 评分 {score}/10，{verdict} (阈值 {min_score}){detail}", ) async def _call_llm( diff --git a/tests/unit/test_llm_score.py b/tests/unit/test_llm_score.py index f3a6cbe..c6e60a2 100644 --- a/tests/unit/test_llm_score.py +++ b/tests/unit/test_llm_score.py @@ -189,3 +189,50 @@ async def test_llm_score_extracts_question_from_sent_message(): user_msg = next((m for m in messages if m["role"] == "user"), None) assert user_msg is not None assert "这是用户的问题" in user_msg["content"] + + +async def test_llm_score_multiturn_uses_last_sent_not_prev_reply(): + """Regression: for multi-turn dialogs the question must come from the LAST + turn's sent_message (the user's question), NOT dialog[-2].reply (the + previous AI reply). The old bug fed two AI replies as a Q&A pair, so the + judge LLM scored everything 0.""" + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5}) + captured_payload = {} + + async def capture_post(url, **kwargs): + captured_payload.update(kwargs.get("json", {})) + return _make_llm_response(score=9.0) + + dialog = [ + _turn("第一轮AI回复", sent_text="第一轮用户问题"), + _turn("第二轮AI回复", sent_text="第二轮用户问题"), + _turn("第三轮AI回复", sent_text="第三轮用户问题"), + ] + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(side_effect=capture_post) + await rule.evaluate(_case(), dialog) + + user_msg = next(m for m in captured_payload["messages"] if m["role"] == "user") + # 问题必须是最后一轮用户发送的问题 + assert "第三轮用户问题" in user_msg["content"] + # 回复必须是最后一轮 AI 回复 + assert "第三轮AI回复" in user_msg["content"] + # 绝不能把上一轮 AI 回复当成"用户问题" + assert "第二轮AI回复" not in user_msg["content"] + + +async def test_llm_score_reason_includes_llm_detail(): + """The rule's reason should surface the judge LLM's own reason for diagnosis.""" + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5}) + mock_resp = _make_llm_response(score=3.0, reason="回复偏离主题") + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(return_value=mock_resp) + result = await rule.evaluate(_case(), [_turn("answer")]) + + assert result.passed is False + assert "回复偏离主题" in result.reason +