AgentEvalTool/tests/unit/test_llm_score.py

"""Unit tests for LlmScoreRule — mocking httpx to avoid real API calls."""

import json
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from agenteval.evaluation.rules.llm_score import LlmScoreRule
from agenteval.models import Case, CaseType, Turn


def _turn(reply_text: str, sent_text: str = "问题", latency_ms: int = 500) -> Turn:
    return Turn(
        id="t1", run_id="r1", case_id="c1", round_index=1,
        sent_message={"msgBody": {"content": sent_text}},
        reply={"msgBody": {"content": reply_text}},
        latency_ms=latency_ms,
    )


def _case() -> Case:
    return Case(id="c1", type=CaseType.SINGLE, messages=["hi"])


def _make_llm_response(score: float, reason: str = "ok") -> MagicMock:
    mock_resp = MagicMock()
    mock_resp.raise_for_status = MagicMock()
    mock_resp.json = MagicMock(return_value={
        "choices": [{"message": {"content": json.dumps({"score": score, "reason": reason})}}]
    })
    return mock_resp


def _make_content_block_response(score: float, reason: str = "ok") -> MagicMock:
    """Simulate Anthropic content-block-array format."""
    mock_resp = MagicMock()
    mock_resp.raise_for_status = MagicMock()
    mock_resp.json = MagicMock(return_value={
        "choices": [{
            "message": {
                "content": [
                    {"type": "text", "text": json.dumps({"score": score, "reason": reason})}
                ]
            }
        }]
    })
    return mock_resp


# ── basic evaluate ────────────────────────────────────────────────────────

async def test_llm_score_no_api_url_fails():
    rule = LlmScoreRule({"criteria": "礼貌"})
    result = await rule.evaluate(_case(), [_turn("回答内容")])
    assert result.passed is False
    assert "api_url" in result.reason


async def test_llm_score_empty_dialog_fails():
    rule = LlmScoreRule({"api_url": "http://mock", "min_score": 7})
    result = await rule.evaluate(_case(), [])
    assert result.passed is False
    assert "无回复" in result.reason


# ── OpenAI format ─────────────────────────────────────────────────────────

async def test_llm_score_passes_above_threshold():
    rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6})
    mock_resp = _make_llm_response(score=8.0, reason="很好")

    with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(return_value=mock_resp)
        result = await rule.evaluate(_case(), [_turn("优质回答")])

    assert result.passed is True
    assert result.score == pytest.approx(0.8)
    assert "8" in result.reason


async def test_llm_score_fails_below_threshold():
    rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 7})
    mock_resp = _make_llm_response(score=4.0, reason="较差")

    with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(return_value=mock_resp)
        result = await rule.evaluate(_case(), [_turn("差劲回答")])

    assert result.passed is False
    assert result.score == pytest.approx(0.4)


async def test_llm_score_clamps_score_to_0_10():
    rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5})
    # API returns out-of-range score
    mock_resp = _make_llm_response(score=12.0)

    with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(return_value=mock_resp)
        result = await rule.evaluate(_case(), [_turn("answer")])

    assert result.score == pytest.approx(1.0)  # clamped 10/10 = 1.0


# ── Anthropic content-block format ────────────────────────────────────────

async def test_llm_score_handles_content_block_array():
    rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6})
    mock_resp = _make_content_block_response(score=7.5)

    with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(return_value=mock_resp)
        result = await rule.evaluate(_case(), [_turn("answer")])

    assert result.passed is True
    assert result.score == pytest.approx(0.75)


# ── JSON fallback parsing ─────────────────────────────────────────────────

async def test_llm_score_parses_json_with_preamble():
    rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6})
    mock_resp = MagicMock()
    mock_resp.raise_for_status = MagicMock()
    mock_resp.json = MagicMock(return_value={
        "choices": [{"message": {"content": 'Sure! Here is the result: {"score": 7, "reason": "decent"}'}}]
    })

    with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(return_value=mock_resp)
        result = await rule.evaluate(_case(), [_turn("answer")])

    assert result.passed is True


# ── error handling ────────────────────────────────────────────────────────

async def test_llm_score_api_error_fails_gracefully():
    rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6})

    with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(side_effect=Exception("connection timeout"))
        result = await rule.evaluate(_case(), [_turn("answer")])

    assert result.passed is False
    assert "timeout" in result.reason.lower() or "LLM" in result.reason


async def test_llm_score_empty_content_fails():
    rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6})
    mock_resp = MagicMock()
    mock_resp.raise_for_status = MagicMock()
    mock_resp.json = MagicMock(return_value={"choices": []})

    with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(return_value=mock_resp)
        result = await rule.evaluate(_case(), [_turn("answer")])

    assert result.passed is False


# ── question extraction ───────────────────────────────────────────────────

async def test_llm_score_extracts_question_from_sent_message():
    """Verifies that sent_message is used as the question when dialog has 1 turn."""
    rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5})
    captured_payload = {}

    mock_resp = _make_llm_response(score=8.0)

    async def capture_post(url, **kwargs):
        captured_payload.update(kwargs.get("json", {}))
        return mock_resp

    with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(side_effect=capture_post)
        await rule.evaluate(_case(), [_turn("答案内容", sent_text="这是用户的问题")])

    # The user_prompt should contain the sent question text
    messages = captured_payload.get("messages", [])
    user_msg = next((m for m in messages if m["role"] == "user"), None)
    assert user_msg is not None
    assert "这是用户的问题" in user_msg["content"]


async def test_llm_score_multiturn_uses_last_sent_not_prev_reply():
    """Regression: for multi-turn dialogs the question must come from the LAST
    turn's sent_message (the user's question), NOT dialog[-2].reply (the
    previous AI reply). The old bug fed two AI replies as a Q&A pair, so the
    judge LLM scored everything 0."""
    rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5})
    captured_payload = {}

    async def capture_post(url, **kwargs):
        captured_payload.update(kwargs.get("json", {}))
        return _make_llm_response(score=9.0)

    dialog = [
        _turn("第一轮AI回复", sent_text="第一轮用户问题"),
        _turn("第二轮AI回复", sent_text="第二轮用户问题"),
        _turn("第三轮AI回复", sent_text="第三轮用户问题"),
    ]

    with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(side_effect=capture_post)
        await rule.evaluate(_case(), dialog)

    user_msg = next(m for m in captured_payload["messages"] if m["role"] == "user")
    # 问题必须是最后一轮用户发送的问题
    assert "第三轮用户问题" in user_msg["content"]
    # 回复必须是最后一轮 AI 回复
    assert "第三轮AI回复" in user_msg["content"]
    # 绝不能把上一轮 AI 回复当成"用户问题"
    assert "第二轮AI回复" not in user_msg["content"]


async def test_llm_score_reason_includes_llm_detail():
    """The rule's reason should surface the judge LLM's own reason for diagnosis."""
    rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5})
    mock_resp = _make_llm_response(score=3.0, reason="回复偏离主题")

    with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(return_value=mock_resp)
        result = await rule.evaluate(_case(), [_turn("answer")])

    assert result.passed is False
    assert "回复偏离主题" in result.reason