fix(llm_score): 修复多轮用例 question 提取错位导致普遍打 0 分
## 现象
多轮/动态用例的 llm_score 规则几乎全部返回 0.0/10 失败,即便被测对象
回复质量很高(实测 400+ 字的专业医疗回复也是 0 分)。
## 根因
evaluate() 里 question 提取逻辑错误:
if len(dialog) >= 2:
question_text = extract_reply_text(dialog[-2].reply) # BUG
dialog[-2].reply 是「上一轮智能体的回复」,被误当成「用户问题」。于是评分
LLM 收到的问答对是:
- "用户问题" = 上一轮 AI 回复
- "智能体回复" = 当前轮 AI 回复
两段都是 AI 说的话、互不相关,评分 LLM 判定牛头不对马嘴 → 打 0 分。
单轮用例因 len(dialog)<2 走 sent_message 提取(正确),故不受影响;
问题只在多轮/dynamic 用例爆发。
## 修复
- question 始终取当前轮 last_turn.sent_message(用户实际发送的问题),
移除错误的 dialog[-2].reply 分支
- reason 增加评分 LLM 自己的理由(parsed["reason"]),便于未来诊断
- 新增回归测试:多轮场景验证 question 来自 last sent_message 而非 prev reply
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
f765bee7a1
commit
c4962ddadf
@ -22,10 +22,11 @@ class LlmScoreRule(EvalRule):
|
||||
last_turn = dialog[-1]
|
||||
reply_text = extract_reply_text(last_turn.reply)
|
||||
|
||||
# 用户问题取自当前轮发送的消息(sent_message),而非上一轮的智能体回复。
|
||||
# 旧逻辑用 dialog[-2].reply 会把「上一轮 AI 回复」误当成「用户问题」,
|
||||
# 导致多轮/动态用例里评分 LLM 收到牛头不对马嘴的问答对,普遍打 0 分。
|
||||
question_text = ""
|
||||
if len(dialog) >= 2:
|
||||
question_text = extract_reply_text(dialog[-2].reply) or ""
|
||||
if not question_text and last_turn.sent_message:
|
||||
if last_turn.sent_message:
|
||||
body = last_turn.sent_message.get("msgBody", "")
|
||||
if isinstance(body, dict):
|
||||
question_text = body.get("content", "")
|
||||
@ -49,10 +50,12 @@ class LlmScoreRule(EvalRule):
|
||||
return RuleResult(passed=False, reason=f"LLM 评分失败: {reason}")
|
||||
|
||||
passed = score >= min_score
|
||||
verdict = "通过" if passed else "未通过"
|
||||
detail = f";{reason}" if reason else ""
|
||||
return RuleResult(
|
||||
passed=passed,
|
||||
score=score / 10.0,
|
||||
reason=f"LLM 评分 {score}/10,{'通过' if passed else '未通过'} (阈值 {min_score})",
|
||||
reason=f"LLM 评分 {score}/10,{verdict} (阈值 {min_score}){detail}",
|
||||
)
|
||||
|
||||
async def _call_llm(
|
||||
|
||||
@ -189,3 +189,50 @@ async def test_llm_score_extracts_question_from_sent_message():
|
||||
user_msg = next((m for m in messages if m["role"] == "user"), None)
|
||||
assert user_msg is not None
|
||||
assert "这是用户的问题" in user_msg["content"]
|
||||
|
||||
|
||||
async def test_llm_score_multiturn_uses_last_sent_not_prev_reply():
|
||||
"""Regression: for multi-turn dialogs the question must come from the LAST
|
||||
turn's sent_message (the user's question), NOT dialog[-2].reply (the
|
||||
previous AI reply). The old bug fed two AI replies as a Q&A pair, so the
|
||||
judge LLM scored everything 0."""
|
||||
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5})
|
||||
captured_payload = {}
|
||||
|
||||
async def capture_post(url, **kwargs):
|
||||
captured_payload.update(kwargs.get("json", {}))
|
||||
return _make_llm_response(score=9.0)
|
||||
|
||||
dialog = [
|
||||
_turn("第一轮AI回复", sent_text="第一轮用户问题"),
|
||||
_turn("第二轮AI回复", sent_text="第二轮用户问题"),
|
||||
_turn("第三轮AI回复", sent_text="第三轮用户问题"),
|
||||
]
|
||||
|
||||
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
||||
instance = MockClient.return_value.__aenter__.return_value
|
||||
instance.post = AsyncMock(side_effect=capture_post)
|
||||
await rule.evaluate(_case(), dialog)
|
||||
|
||||
user_msg = next(m for m in captured_payload["messages"] if m["role"] == "user")
|
||||
# 问题必须是最后一轮用户发送的问题
|
||||
assert "第三轮用户问题" in user_msg["content"]
|
||||
# 回复必须是最后一轮 AI 回复
|
||||
assert "第三轮AI回复" in user_msg["content"]
|
||||
# 绝不能把上一轮 AI 回复当成"用户问题"
|
||||
assert "第二轮AI回复" not in user_msg["content"]
|
||||
|
||||
|
||||
async def test_llm_score_reason_includes_llm_detail():
|
||||
"""The rule's reason should surface the judge LLM's own reason for diagnosis."""
|
||||
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5})
|
||||
mock_resp = _make_llm_response(score=3.0, reason="回复偏离主题")
|
||||
|
||||
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
||||
instance = MockClient.return_value.__aenter__.return_value
|
||||
instance.post = AsyncMock(return_value=mock_resp)
|
||||
result = await rule.evaluate(_case(), [_turn("answer")])
|
||||
|
||||
assert result.passed is False
|
||||
assert "回复偏离主题" in result.reason
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user