c4962ddadf
## 现象
多轮/动态用例的 llm_score 规则几乎全部返回 0.0/10 失败，即便被测对象
回复质量很高（实测 400+ 字的专业医疗回复也是 0 分）。
## 根因
evaluate() 里 question 提取逻辑错误：
if len(dialog) >= 2:
question_text = extract_reply_text(dialog[-2].reply) # BUG
dialog[-2].reply 是「上一轮智能体的回复」，被误当成「用户问题」。于是评分
LLM 收到的问答对是：
- "用户问题" = 上一轮 AI 回复
- "智能体回复" = 当前轮 AI 回复
两段都是 AI 说的话、互不相关，评分 LLM 判定牛头不对马嘴 → 打 0 分。
单轮用例因 len(dialog)<2 走 sent_message 提取（正确），故不受影响；
问题只在多轮/dynamic 用例爆发。
## 修复
- question 始终取当前轮 last_turn.sent_message（用户实际发送的问题），
移除错误的 dialog[-2].reply 分支
- reason 增加评分 LLM 自己的理由（parsed["reason"]），便于未来诊断
- 新增回归测试：多轮场景验证 question 来自 last sent_message 而非 prev reply
Co-Authored-By: Claude <noreply@anthropic.com>
239 lines
10 KiB
Python
239 lines
10 KiB
Python
"""Unit tests for LlmScoreRule — mocking httpx to avoid real API calls."""
|
|
|
|
import json
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from agenteval.evaluation.rules.llm_score import LlmScoreRule
|
|
from agenteval.models import Case, CaseType, Turn
|
|
|
|
|
|
def _turn(reply_text: str, sent_text: str = "问题", latency_ms: int = 500) -> Turn:
|
|
return Turn(
|
|
id="t1", run_id="r1", case_id="c1", round_index=1,
|
|
sent_message={"msgBody": {"content": sent_text}},
|
|
reply={"msgBody": {"content": reply_text}},
|
|
latency_ms=latency_ms,
|
|
)
|
|
|
|
|
|
def _case() -> Case:
|
|
return Case(id="c1", type=CaseType.SINGLE, messages=["hi"])
|
|
|
|
|
|
def _make_llm_response(score: float, reason: str = "ok") -> MagicMock:
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
mock_resp.json = MagicMock(return_value={
|
|
"choices": [{"message": {"content": json.dumps({"score": score, "reason": reason})}}]
|
|
})
|
|
return mock_resp
|
|
|
|
|
|
def _make_content_block_response(score: float, reason: str = "ok") -> MagicMock:
|
|
"""Simulate Anthropic content-block-array format."""
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
mock_resp.json = MagicMock(return_value={
|
|
"choices": [{
|
|
"message": {
|
|
"content": [
|
|
{"type": "text", "text": json.dumps({"score": score, "reason": reason})}
|
|
]
|
|
}
|
|
}]
|
|
})
|
|
return mock_resp
|
|
|
|
|
|
# ── basic evaluate ────────────────────────────────────────────────────────
|
|
|
|
async def test_llm_score_no_api_url_fails():
|
|
rule = LlmScoreRule({"criteria": "礼貌"})
|
|
result = await rule.evaluate(_case(), [_turn("回答内容")])
|
|
assert result.passed is False
|
|
assert "api_url" in result.reason
|
|
|
|
|
|
async def test_llm_score_empty_dialog_fails():
|
|
rule = LlmScoreRule({"api_url": "http://mock", "min_score": 7})
|
|
result = await rule.evaluate(_case(), [])
|
|
assert result.passed is False
|
|
assert "无回复" in result.reason
|
|
|
|
|
|
# ── OpenAI format ─────────────────────────────────────────────────────────
|
|
|
|
async def test_llm_score_passes_above_threshold():
|
|
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6})
|
|
mock_resp = _make_llm_response(score=8.0, reason="很好")
|
|
|
|
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(return_value=mock_resp)
|
|
result = await rule.evaluate(_case(), [_turn("优质回答")])
|
|
|
|
assert result.passed is True
|
|
assert result.score == pytest.approx(0.8)
|
|
assert "8" in result.reason
|
|
|
|
|
|
async def test_llm_score_fails_below_threshold():
|
|
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 7})
|
|
mock_resp = _make_llm_response(score=4.0, reason="较差")
|
|
|
|
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(return_value=mock_resp)
|
|
result = await rule.evaluate(_case(), [_turn("差劲回答")])
|
|
|
|
assert result.passed is False
|
|
assert result.score == pytest.approx(0.4)
|
|
|
|
|
|
async def test_llm_score_clamps_score_to_0_10():
|
|
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5})
|
|
# API returns out-of-range score
|
|
mock_resp = _make_llm_response(score=12.0)
|
|
|
|
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(return_value=mock_resp)
|
|
result = await rule.evaluate(_case(), [_turn("answer")])
|
|
|
|
assert result.score == pytest.approx(1.0) # clamped 10/10 = 1.0
|
|
|
|
|
|
# ── Anthropic content-block format ────────────────────────────────────────
|
|
|
|
async def test_llm_score_handles_content_block_array():
|
|
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6})
|
|
mock_resp = _make_content_block_response(score=7.5)
|
|
|
|
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(return_value=mock_resp)
|
|
result = await rule.evaluate(_case(), [_turn("answer")])
|
|
|
|
assert result.passed is True
|
|
assert result.score == pytest.approx(0.75)
|
|
|
|
|
|
# ── JSON fallback parsing ─────────────────────────────────────────────────
|
|
|
|
async def test_llm_score_parses_json_with_preamble():
|
|
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6})
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
mock_resp.json = MagicMock(return_value={
|
|
"choices": [{"message": {"content": 'Sure! Here is the result: {"score": 7, "reason": "decent"}'}}]
|
|
})
|
|
|
|
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(return_value=mock_resp)
|
|
result = await rule.evaluate(_case(), [_turn("answer")])
|
|
|
|
assert result.passed is True
|
|
|
|
|
|
# ── error handling ────────────────────────────────────────────────────────
|
|
|
|
async def test_llm_score_api_error_fails_gracefully():
|
|
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6})
|
|
|
|
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(side_effect=Exception("connection timeout"))
|
|
result = await rule.evaluate(_case(), [_turn("answer")])
|
|
|
|
assert result.passed is False
|
|
assert "timeout" in result.reason.lower() or "LLM" in result.reason
|
|
|
|
|
|
async def test_llm_score_empty_content_fails():
|
|
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6})
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
mock_resp.json = MagicMock(return_value={"choices": []})
|
|
|
|
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(return_value=mock_resp)
|
|
result = await rule.evaluate(_case(), [_turn("answer")])
|
|
|
|
assert result.passed is False
|
|
|
|
|
|
# ── question extraction ───────────────────────────────────────────────────
|
|
|
|
async def test_llm_score_extracts_question_from_sent_message():
|
|
"""Verifies that sent_message is used as the question when dialog has 1 turn."""
|
|
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5})
|
|
captured_payload = {}
|
|
|
|
mock_resp = _make_llm_response(score=8.0)
|
|
|
|
async def capture_post(url, **kwargs):
|
|
captured_payload.update(kwargs.get("json", {}))
|
|
return mock_resp
|
|
|
|
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(side_effect=capture_post)
|
|
await rule.evaluate(_case(), [_turn("答案内容", sent_text="这是用户的问题")])
|
|
|
|
# The user_prompt should contain the sent question text
|
|
messages = captured_payload.get("messages", [])
|
|
user_msg = next((m for m in messages if m["role"] == "user"), None)
|
|
assert user_msg is not None
|
|
assert "这是用户的问题" in user_msg["content"]
|
|
|
|
|
|
async def test_llm_score_multiturn_uses_last_sent_not_prev_reply():
|
|
"""Regression: for multi-turn dialogs the question must come from the LAST
|
|
turn's sent_message (the user's question), NOT dialog[-2].reply (the
|
|
previous AI reply). The old bug fed two AI replies as a Q&A pair, so the
|
|
judge LLM scored everything 0."""
|
|
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5})
|
|
captured_payload = {}
|
|
|
|
async def capture_post(url, **kwargs):
|
|
captured_payload.update(kwargs.get("json", {}))
|
|
return _make_llm_response(score=9.0)
|
|
|
|
dialog = [
|
|
_turn("第一轮AI回复", sent_text="第一轮用户问题"),
|
|
_turn("第二轮AI回复", sent_text="第二轮用户问题"),
|
|
_turn("第三轮AI回复", sent_text="第三轮用户问题"),
|
|
]
|
|
|
|
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(side_effect=capture_post)
|
|
await rule.evaluate(_case(), dialog)
|
|
|
|
user_msg = next(m for m in captured_payload["messages"] if m["role"] == "user")
|
|
# 问题必须是最后一轮用户发送的问题
|
|
assert "第三轮用户问题" in user_msg["content"]
|
|
# 回复必须是最后一轮 AI 回复
|
|
assert "第三轮AI回复" in user_msg["content"]
|
|
# 绝不能把上一轮 AI 回复当成"用户问题"
|
|
assert "第二轮AI回复" not in user_msg["content"]
|
|
|
|
|
|
async def test_llm_score_reason_includes_llm_detail():
|
|
"""The rule's reason should surface the judge LLM's own reason for diagnosis."""
|
|
rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5})
|
|
mock_resp = _make_llm_response(score=3.0, reason="回复偏离主题")
|
|
|
|
with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(return_value=mock_resp)
|
|
result = await rule.evaluate(_case(), [_turn("answer")])
|
|
|
|
assert result.passed is False
|
|
assert "回复偏离主题" in result.reason
|
|
|