"""Unit tests for HttpChannel, OpenClawChannel, and async rule evaluation.""" import asyncio from unittest.mock import AsyncMock, MagicMock, patch import pytest from agenteval.channels.http import HttpChannel, _get_path from agenteval.channels.openclaw import OpenClawChannel from agenteval.channels.base import SendResult from agenteval.evaluation.rules.keyword import KeywordMatchRule from agenteval.evaluation.rules.response_time import ResponseTimeRule from agenteval.models import Case, CaseType, Expectation, Turn # ── _get_path helper ───────────────────────────────────────────────────── def test_get_path_simple(): assert _get_path({"id": "abc"}, "id") == "abc" def test_get_path_nested(): assert _get_path({"a": {"b": {"c": 42}}}, "a.b.c") == 42 def test_get_path_missing(): assert _get_path({"a": 1}, "a.b") is None def test_get_path_none_data(): assert _get_path(None, "x") is None def test_get_path_list_index(): assert _get_path({"items": ["x", "y"]}, "items.1") == "y" # ── HttpChannel ────────────────────────────────────────────────────────── def _make_channel(**extra) -> HttpChannel: config = { "send_url": "http://mock/send", "reply_url": "http://mock/reply/{msg_id}", "health_url": "http://mock/health", **extra, } return HttpChannel(config) async def test_http_health_check_ok(): ch = _make_channel() mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.raise_for_status = MagicMock() with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)): result = await ch.health_check() assert result.ok is True assert "200" in result.message async def test_http_health_check_fail(): ch = _make_channel() with patch.object(ch._client, "get", new=AsyncMock(side_effect=Exception("conn refused"))): result = await ch.health_check() assert result.ok is False assert "conn refused" in result.message async def test_http_send_extracts_msg_id(): ch = _make_channel(msg_id_path="data.id") mock_resp = MagicMock() mock_resp.raise_for_status = MagicMock() mock_resp.json = MagicMock(return_value={"data": {"id": "msg-42"}}) with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)): result = await ch.send("hello") assert result.ok is True assert result.question_msg_id == "msg-42" async def test_http_send_failure(): ch = _make_channel() with patch.object(ch._client, "post", new=AsyncMock(side_effect=Exception("timeout"))): result = await ch.send("hi") assert result.ok is False assert "timeout" in result.error async def test_http_poll_reply_found(): ch = _make_channel(reply_path="answer") call_count = {"n": 0} async def mock_get(url, **kwargs): call_count["n"] += 1 mock_resp = MagicMock() mock_resp.raise_for_status = MagicMock() mock_resp.json = MagicMock(return_value={"answer": "Hello world"}) return mock_resp with patch.object(ch._client, "get", new=mock_get): reply = await ch.poll_reply("msg-1", timeout=5.0) assert reply is not None assert reply.content == "Hello world" assert reply.question_msg_id == "msg-1" async def test_http_poll_reply_timeout(): ch = _make_channel(reply_path="missing_field", poll_interval=0.05) async def mock_get(url, **kwargs): mock_resp = MagicMock() mock_resp.raise_for_status = MagicMock() mock_resp.json = MagicMock(return_value={}) # field not present return mock_resp with patch.object(ch._client, "get", new=mock_get): reply = await ch.poll_reply("msg-1", timeout=0.15, poll_interval=0.05) assert reply is None async def test_http_poll_reply_readiness_flag(): ch = _make_channel(reply_path="text", reply_ready_path="ready") responses = [ {"ready": False, "text": "not ready"}, {"ready": True, "text": "final answer"}, ] call_idx = {"n": 0} async def mock_get(url, **kwargs): mock_resp = MagicMock() mock_resp.raise_for_status = MagicMock() mock_resp.json = MagicMock(return_value=responses[min(call_idx["n"], 1)]) call_idx["n"] += 1 return mock_resp with patch.object(ch._client, "get", new=mock_get): reply = await ch.poll_reply("msg-1", timeout=5.0, poll_interval=0.05) assert reply is not None assert reply.content == "final answer" # ── Async rule evaluation ──────────────────────────────────────────────── def _make_turn(reply_text: str, latency_ms: int = 100) -> Turn: return Turn( id="t1", run_id="r1", case_id="c1", round_index=1, reply={"msgBody": {"content": reply_text}}, latency_ms=latency_ms, ) def _make_case(*, keywords: list[str] | None = None, max_ms: int | None = None) -> Case: return Case( id="c1", type=CaseType.SINGLE, messages=["hi"], expectations=Expectation( keywords_include=keywords or [], response_time_max_ms=max_ms, ), ) async def test_keyword_rule_is_async_and_passes(): rule = KeywordMatchRule({"keywords": ["hello", "world"]}) turn = _make_turn("hello world here") result = await rule.evaluate(_make_case(), [turn]) assert result.passed is True assert result.score == 1.0 async def test_keyword_rule_fails_missing_keyword(): rule = KeywordMatchRule({"keywords": ["missing"]}) turn = _make_turn("some other text") result = await rule.evaluate(_make_case(), [turn]) assert result.passed is False assert "missing" in result.reason async def test_keyword_rule_fails_excluded_keyword(): rule = KeywordMatchRule({"exclude_keywords": ["banned"]}) turn = _make_turn("this is banned content") result = await rule.evaluate(_make_case(), [turn]) assert result.passed is False async def test_response_time_rule_is_async_and_passes(): rule = ResponseTimeRule({"max_ms": 500}) turn = _make_turn("ok", latency_ms=200) result = await rule.evaluate(_make_case(), [turn]) assert result.passed is True async def test_response_time_rule_fails_over_threshold(): rule = ResponseTimeRule({"max_ms": 100}) turn = _make_turn("slow response", latency_ms=5000) result = await rule.evaluate(_make_case(), [turn]) assert result.passed is False assert "5000ms" in result.reason async def test_response_time_uses_expectation_fallback(): rule = ResponseTimeRule({}) # no max_ms in params case = _make_case(max_ms=200) turn = _make_turn("ok", latency_ms=100) result = await rule.evaluate(case, [turn]) assert result.passed is True async def test_rules_empty_dialog_fail(): keyword_rule = KeywordMatchRule({"keywords": ["x"]}) rt_rule = ResponseTimeRule({"max_ms": 1000}) case = _make_case() for rule in [keyword_rule, rt_rule]: result = await rule.evaluate(case, []) assert result.passed is False assert "无回复" in result.reason # ── OpenClawChannel ────────────────────────────────────────────────────── def _make_openclaw_channel(**extra) -> OpenClawChannel: config = {"base_url": "http://mock-openclaw:18789", "auth_token": "test-token", **extra} with patch("agenteval.channels.openclaw.get_settings") as mock_settings: mock_settings.return_value.openclaw_upstream = "http://default:18789" mock_settings.return_value.openclaw_auth_token = "default-token" return OpenClawChannel(config) async def test_openclaw_health_check_ok(): ch = _make_openclaw_channel() mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.raise_for_status = MagicMock() with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)): result = await ch.health_check() assert result.ok is True assert "OpenClaw" in result.message async def test_openclaw_health_check_fail(): ch = _make_openclaw_channel() with patch.object(ch._client, "get", new=AsyncMock(side_effect=Exception("conn refused"))): result = await ch.health_check() assert result.ok is False async def test_openclaw_send_ok(): ch = _make_openclaw_channel() mock_resp = MagicMock() mock_resp.raise_for_status = MagicMock() mock_resp.json = MagicMock(return_value={"id": "chat-msg-1", "choices": [{"message": {"content": "reply"}}]}) with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)): result = await ch.send("hello") assert result.ok is True assert result.question_msg_id == "chat-msg-1" async def test_openclaw_send_failure(): ch = _make_openclaw_channel() with patch.object(ch._client, "post", new=AsyncMock(side_effect=Exception("timeout"))): result = await ch.send("hi") assert result.ok is False async def test_openclaw_poll_reply_found(): ch = _make_openclaw_channel() mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.json = MagicMock(return_value={"choices": [{"message": {"content": "assistant reply"}}]}) with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)): reply = await ch.poll_reply("chat-msg-1", timeout=5.0) assert reply is not None assert reply.content == "assistant reply" async def test_openclaw_poll_reply_timeout(): ch = _make_openclaw_channel(poll_interval=0.05) mock_resp = MagicMock() mock_resp.status_code = 404 mock_resp.json = MagicMock(return_value={}) with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)): reply = await ch.poll_reply("chat-msg-1", timeout=0.15, poll_interval=0.05) assert reply is None async def test_openclaw_uses_default_settings(): """When config has no base_url or auth_token, fall back to settings defaults.""" with patch("agenteval.channels.openclaw.get_settings") as mock_settings: mock_settings.return_value.openclaw_upstream = "http://default:18789" mock_settings.return_value.openclaw_auth_token = "default-token" ch = OpenClawChannel({}) assert ch.base_url == "http://default:18789" assert ch.auth_token == "default-token"