diff --git a/backend/agenteval/evaluation/rules/llm_score.py b/backend/agenteval/evaluation/rules/llm_score.py index 9193352..ec7b810 100644 --- a/backend/agenteval/evaluation/rules/llm_score.py +++ b/backend/agenteval/evaluation/rules/llm_score.py @@ -22,10 +22,11 @@ class LlmScoreRule(EvalRule): last_turn = dialog[-1] reply_text = extract_reply_text(last_turn.reply) + # 用户问题取自当前轮发送的消息(sent_message),而非上一轮的智能体回复。 + # 旧逻辑用 dialog[-2].reply 会把「上一轮 AI 回复」误当成「用户问题」, + # 导致多轮/动态用例里评分 LLM 收到牛头不对马嘴的问答对,普遍打 0 分。 question_text = "" - if len(dialog) >= 2: - question_text = extract_reply_text(dialog[-2].reply) or "" - if not question_text and last_turn.sent_message: + if last_turn.sent_message: body = last_turn.sent_message.get("msgBody", "") if isinstance(body, dict): question_text = body.get("content", "") @@ -49,10 +50,12 @@ class LlmScoreRule(EvalRule): return RuleResult(passed=False, reason=f"LLM 评分失败: {reason}") passed = score >= min_score + verdict = "通过" if passed else "未通过" + detail = f";{reason}" if reason else "" return RuleResult( passed=passed, score=score / 10.0, - reason=f"LLM 评分 {score}/10,{'通过' if passed else '未通过'} (阈值 {min_score})", + reason=f"LLM 评分 {score}/10,{verdict} (阈值 {min_score}){detail}", ) async def _call_llm( diff --git a/tests/unit/test_llm_score.py b/tests/unit/test_llm_score.py index f3a6cbe..c6e60a2 100644 --- a/tests/unit/test_llm_score.py +++ b/tests/unit/test_llm_score.py @@ -189,3 +189,50 @@ async def test_llm_score_extracts_question_from_sent_message(): user_msg = next((m for m in messages if m["role"] == "user"), None) assert user_msg is not None assert "这是用户的问题" in user_msg["content"] + + +async def test_llm_score_multiturn_uses_last_sent_not_prev_reply(): + """Regression: for multi-turn dialogs the question must come from the LAST + turn's sent_message (the user's question), NOT dialog[-2].reply (the + previous AI reply). The old bug fed two AI replies as a Q&A pair, so the + judge LLM scored everything 0.""" + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5}) + captured_payload = {} + + async def capture_post(url, **kwargs): + captured_payload.update(kwargs.get("json", {})) + return _make_llm_response(score=9.0) + + dialog = [ + _turn("第一轮AI回复", sent_text="第一轮用户问题"), + _turn("第二轮AI回复", sent_text="第二轮用户问题"), + _turn("第三轮AI回复", sent_text="第三轮用户问题"), + ] + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(side_effect=capture_post) + await rule.evaluate(_case(), dialog) + + user_msg = next(m for m in captured_payload["messages"] if m["role"] == "user") + # 问题必须是最后一轮用户发送的问题 + assert "第三轮用户问题" in user_msg["content"] + # 回复必须是最后一轮 AI 回复 + assert "第三轮AI回复" in user_msg["content"] + # 绝不能把上一轮 AI 回复当成"用户问题" + assert "第二轮AI回复" not in user_msg["content"] + + +async def test_llm_score_reason_includes_llm_detail(): + """The rule's reason should surface the judge LLM's own reason for diagnosis.""" + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5}) + mock_resp = _make_llm_response(score=3.0, reason="回复偏离主题") + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(return_value=mock_resp) + result = await rule.evaluate(_case(), [_turn("answer")]) + + assert result.passed is False + assert "回复偏离主题" in result.reason +