312 lines
11 KiB
Python
312 lines
11 KiB
Python
"""Unit tests for HttpChannel, OpenClawChannel, and async rule evaluation."""
|
|
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import httpx
|
|
import pytest
|
|
from agenteval.channels.base import ExchangeStatus
|
|
from agenteval.channels.http import HttpChannel, _get_path
|
|
from agenteval.channels.openclaw import OpenClawChannel
|
|
from agenteval.evaluation.rules.keyword import KeywordMatchRule
|
|
from agenteval.evaluation.rules.response_time import ResponseTimeRule
|
|
from agenteval.models import Case, CaseType, Expectation, Turn
|
|
|
|
# ── _get_path helper ─────────────────────────────────────────────────────
|
|
|
|
def test_get_path_simple():
|
|
assert _get_path({"id": "abc"}, "id") == "abc"
|
|
|
|
|
|
def test_get_path_nested():
|
|
assert _get_path({"a": {"b": {"c": 42}}}, "a.b.c") == 42
|
|
|
|
|
|
def test_get_path_missing():
|
|
assert _get_path({"a": 1}, "a.b") is None
|
|
|
|
|
|
def test_get_path_none_data():
|
|
assert _get_path(None, "x") is None
|
|
|
|
|
|
def test_get_path_list_index():
|
|
assert _get_path({"items": ["x", "y"]}, "items.1") == "y"
|
|
|
|
|
|
# ── HttpChannel ──────────────────────────────────────────────────────────
|
|
|
|
def _make_channel(**extra) -> HttpChannel:
|
|
config = {
|
|
"send_url": "http://mock/send",
|
|
"reply_url": "http://mock/reply/{msg_id}",
|
|
"health_url": "http://mock/health",
|
|
**extra,
|
|
}
|
|
return HttpChannel(config)
|
|
|
|
|
|
async def test_http_health_check_ok():
|
|
ch = _make_channel()
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.raise_for_status = MagicMock()
|
|
with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)):
|
|
result = await ch.health_check()
|
|
assert result.ok is True
|
|
assert "200" in result.message
|
|
|
|
|
|
async def test_http_health_check_fail():
|
|
ch = _make_channel()
|
|
with patch.object(ch._client, "get", new=AsyncMock(side_effect=Exception("conn refused"))):
|
|
result = await ch.health_check()
|
|
assert result.ok is False
|
|
assert "conn refused" in result.message
|
|
|
|
|
|
async def test_http_send_extracts_msg_id():
|
|
ch = _make_channel(msg_id_path="data.id")
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
mock_resp.json = MagicMock(return_value={"data": {"id": "msg-42"}})
|
|
with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)):
|
|
result = await ch._send("hello")
|
|
assert result.ok is True
|
|
assert result.question_msg_id == "msg-42"
|
|
|
|
|
|
async def test_http_send_failure():
|
|
ch = _make_channel()
|
|
with patch.object(ch._client, "post", new=AsyncMock(side_effect=httpx.ConnectError("timeout"))):
|
|
result = await ch._send("hi")
|
|
assert result.ok is False
|
|
assert "timeout" in result.error
|
|
|
|
|
|
async def test_http_exchange_classifies_poll_transport_failure():
|
|
ch = _make_channel()
|
|
sent = MagicMock()
|
|
sent.raise_for_status = MagicMock()
|
|
sent.json = MagicMock(return_value={"id": "msg-1"})
|
|
with (
|
|
patch.object(ch._client, "post", new=AsyncMock(return_value=sent)),
|
|
patch.object(ch._client, "get", new=AsyncMock(side_effect=httpx.ConnectError("offline"))),
|
|
):
|
|
outcome = await ch.exchange("hi")
|
|
|
|
assert outcome.status is ExchangeStatus.POLL_FAILED
|
|
assert "offline" in outcome.reason
|
|
|
|
|
|
async def test_http_exchange_does_not_mask_adapter_programming_errors():
|
|
ch = _make_channel()
|
|
with patch.object(ch._client, "post", new=AsyncMock(side_effect=RuntimeError("adapter bug"))):
|
|
with pytest.raises(RuntimeError, match="adapter bug"):
|
|
await ch.exchange("hi")
|
|
|
|
|
|
async def test_http_poll_reply_found():
|
|
ch = _make_channel(reply_path="answer")
|
|
call_count = {"n": 0}
|
|
|
|
async def mock_get(url, **kwargs):
|
|
call_count["n"] += 1
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
mock_resp.json = MagicMock(return_value={"answer": "Hello world"})
|
|
return mock_resp
|
|
|
|
with patch.object(ch._client, "get", new=mock_get):
|
|
reply = await ch._poll_reply("msg-1", timeout=5.0)
|
|
assert reply is not None
|
|
assert reply.content == "Hello world"
|
|
assert reply.question_msg_id == "msg-1"
|
|
|
|
|
|
async def test_http_poll_reply_timeout():
|
|
ch = _make_channel(reply_path="missing_field", poll_interval=0.05)
|
|
|
|
async def mock_get(url, **kwargs):
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
mock_resp.json = MagicMock(return_value={}) # field not present
|
|
return mock_resp
|
|
|
|
with patch.object(ch._client, "get", new=mock_get):
|
|
reply = await ch._poll_reply("msg-1", timeout=0.15, poll_interval=0.05)
|
|
assert reply is None
|
|
|
|
|
|
async def test_http_poll_reply_readiness_flag():
|
|
ch = _make_channel(reply_path="text", reply_ready_path="ready")
|
|
responses = [
|
|
{"ready": False, "text": "not ready"},
|
|
{"ready": True, "text": "final answer"},
|
|
]
|
|
call_idx = {"n": 0}
|
|
|
|
async def mock_get(url, **kwargs):
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
mock_resp.json = MagicMock(return_value=responses[min(call_idx["n"], 1)])
|
|
call_idx["n"] += 1
|
|
return mock_resp
|
|
|
|
with patch.object(ch._client, "get", new=mock_get):
|
|
reply = await ch._poll_reply("msg-1", timeout=5.0, poll_interval=0.05)
|
|
assert reply is not None
|
|
assert reply.content == "final answer"
|
|
|
|
|
|
# ── Async rule evaluation ────────────────────────────────────────────────
|
|
|
|
def _make_turn(reply_text: str, latency_ms: int = 100) -> Turn:
|
|
return Turn(
|
|
id="t1", run_id="r1", case_id="c1", round_index=1,
|
|
reply={"msgBody": {"content": reply_text}},
|
|
latency_ms=latency_ms,
|
|
)
|
|
|
|
|
|
def _make_case(*, keywords: list[str] | None = None, max_ms: int | None = None) -> Case:
|
|
return Case(
|
|
id="c1", type=CaseType.SINGLE, messages=["hi"],
|
|
expectations=Expectation(
|
|
keywords_include=keywords or [],
|
|
response_time_max_ms=max_ms,
|
|
),
|
|
)
|
|
|
|
|
|
async def test_keyword_rule_is_async_and_passes():
|
|
rule = KeywordMatchRule({"keywords": ["hello", "world"]})
|
|
turn = _make_turn("hello world here")
|
|
result = await rule.evaluate(_make_case(), [turn])
|
|
assert result.passed is True
|
|
assert result.score == 1.0
|
|
|
|
|
|
async def test_keyword_rule_fails_missing_keyword():
|
|
rule = KeywordMatchRule({"keywords": ["missing"]})
|
|
turn = _make_turn("some other text")
|
|
result = await rule.evaluate(_make_case(), [turn])
|
|
assert result.passed is False
|
|
assert "missing" in result.reason
|
|
|
|
|
|
async def test_keyword_rule_fails_excluded_keyword():
|
|
rule = KeywordMatchRule({"exclude_keywords": ["banned"]})
|
|
turn = _make_turn("this is banned content")
|
|
result = await rule.evaluate(_make_case(), [turn])
|
|
assert result.passed is False
|
|
|
|
|
|
async def test_response_time_rule_is_async_and_passes():
|
|
rule = ResponseTimeRule({"max_ms": 500})
|
|
turn = _make_turn("ok", latency_ms=200)
|
|
result = await rule.evaluate(_make_case(), [turn])
|
|
assert result.passed is True
|
|
|
|
|
|
async def test_response_time_rule_fails_over_threshold():
|
|
rule = ResponseTimeRule({"max_ms": 100})
|
|
turn = _make_turn("slow response", latency_ms=5000)
|
|
result = await rule.evaluate(_make_case(), [turn])
|
|
assert result.passed is False
|
|
assert "5000ms" in result.reason
|
|
|
|
|
|
async def test_response_time_uses_expectation_fallback():
|
|
rule = ResponseTimeRule({}) # no max_ms in params
|
|
case = _make_case(max_ms=200)
|
|
turn = _make_turn("ok", latency_ms=100)
|
|
result = await rule.evaluate(case, [turn])
|
|
assert result.passed is True
|
|
|
|
|
|
async def test_rules_empty_dialog_fail():
|
|
keyword_rule = KeywordMatchRule({"keywords": ["x"]})
|
|
rt_rule = ResponseTimeRule({"max_ms": 1000})
|
|
case = _make_case()
|
|
for rule in [keyword_rule, rt_rule]:
|
|
result = await rule.evaluate(case, [])
|
|
assert result.passed is False
|
|
assert "无回复" in result.reason
|
|
|
|
|
|
# ── OpenClawChannel ──────────────────────────────────────────────────────
|
|
|
|
def _make_openclaw_channel(**extra) -> OpenClawChannel:
|
|
config = {"base_url": "http://mock-openclaw:18789", "auth_token": "test-token", **extra}
|
|
with patch("agenteval.channels.openclaw.get_settings") as mock_settings:
|
|
mock_settings.return_value.openclaw_upstream = "http://default:18789"
|
|
mock_settings.return_value.openclaw_auth_token = "default-token"
|
|
return OpenClawChannel(config)
|
|
|
|
|
|
async def test_openclaw_health_check_ok():
|
|
ch = _make_openclaw_channel()
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.raise_for_status = MagicMock()
|
|
with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)):
|
|
result = await ch.health_check()
|
|
assert result.ok is True
|
|
assert "OpenClaw" in result.message
|
|
|
|
|
|
async def test_openclaw_health_check_fail():
|
|
ch = _make_openclaw_channel()
|
|
with patch.object(ch._client, "get", new=AsyncMock(side_effect=Exception("conn refused"))):
|
|
result = await ch.health_check()
|
|
assert result.ok is False
|
|
|
|
|
|
async def test_openclaw_send_ok():
|
|
ch = _make_openclaw_channel()
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
mock_resp.json = MagicMock(return_value={"id": "chat-msg-1", "choices": [{"message": {"content": "reply"}}]})
|
|
with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)):
|
|
result = await ch._send("hello")
|
|
assert result.ok is True
|
|
assert result.question_msg_id == "chat-msg-1"
|
|
|
|
|
|
async def test_openclaw_send_failure():
|
|
ch = _make_openclaw_channel()
|
|
with patch.object(ch._client, "post", new=AsyncMock(side_effect=httpx.ConnectError("timeout"))):
|
|
result = await ch._send("hi")
|
|
assert result.ok is False
|
|
|
|
|
|
async def test_openclaw_poll_reply_found():
|
|
ch = _make_openclaw_channel()
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.json = MagicMock(return_value={"choices": [{"message": {"content": "assistant reply"}}]})
|
|
with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)):
|
|
reply = await ch._poll_reply("chat-msg-1", timeout=5.0)
|
|
assert reply is not None
|
|
assert reply.content == "assistant reply"
|
|
|
|
|
|
async def test_openclaw_poll_reply_timeout():
|
|
ch = _make_openclaw_channel(poll_interval=0.05)
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 404
|
|
mock_resp.json = MagicMock(return_value={})
|
|
with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)):
|
|
reply = await ch._poll_reply("chat-msg-1", timeout=0.15, poll_interval=0.05)
|
|
assert reply is None
|
|
|
|
|
|
async def test_openclaw_uses_default_settings():
|
|
"""When config has no base_url or auth_token, fall back to settings defaults."""
|
|
with patch("agenteval.channels.openclaw.get_settings") as mock_settings:
|
|
mock_settings.return_value.openclaw_upstream = "http://default:18789"
|
|
mock_settings.return_value.openclaw_auth_token = "default-token"
|
|
ch = OpenClawChannel({})
|
|
assert ch.base_url == "http://default:18789"
|
|
assert ch.auth_token == "default-token"
|