148 lines
5.2 KiB
Python
148 lines
5.2 KiB
Python
"""Contract tests for the complete channel exchange outcome."""
|
|
|
|
import pytest
|
|
from agenteval.channels.base import (
|
|
ChannelHealth,
|
|
ChannelTransportError,
|
|
EvalChannel,
|
|
ExchangeOutcome,
|
|
ExchangeStatus,
|
|
Reply,
|
|
SendResult,
|
|
normalize_reply_text,
|
|
)
|
|
|
|
|
|
class ContractChannel(EvalChannel):
|
|
def __init__(self, *, send_result=None, reply=None, poll_error=None):
|
|
self.send_result = send_result or SendResult(ok=True, question_msg_id="question-7")
|
|
self.reply = reply
|
|
self.poll_error = poll_error
|
|
self.events: list[str] = []
|
|
|
|
async def health_check(self):
|
|
return ChannelHealth(ok=True)
|
|
|
|
async def _send(self, content, **kwargs):
|
|
self.events.append("send")
|
|
return self.send_result
|
|
|
|
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
|
|
self.events.append("poll")
|
|
if self.poll_error:
|
|
raise self.poll_error
|
|
return self.reply
|
|
|
|
|
|
def test_success_normalizes_reply_and_preserves_exchange_metadata():
|
|
diagnostic = {"provider": "tutu", "request_id": "req-7"}
|
|
|
|
outcome = ExchangeOutcome.succeeded(
|
|
correlation_id="question-7",
|
|
reply={"msgBody": {"content": "你好"}},
|
|
latency_ms=128,
|
|
diagnostic=diagnostic,
|
|
)
|
|
|
|
assert outcome.status is ExchangeStatus.SUCCESS
|
|
assert outcome.ok is True
|
|
assert outcome.expected_failure is False
|
|
assert outcome.correlation_id == "question-7"
|
|
assert outcome.reply_text == "你好"
|
|
assert outcome.latency_ms == 128
|
|
assert outcome.diagnostic is diagnostic
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("outcome", "status"),
|
|
[
|
|
(ExchangeOutcome.send_failed("connection refused"), ExchangeStatus.SEND_FAILED),
|
|
(ExchangeOutcome.reply_timeout(correlation_id="question-7", latency_ms=30_000), ExchangeStatus.REPLY_TIMEOUT),
|
|
(
|
|
ExchangeOutcome.poll_failed("upstream returned 502", correlation_id="question-7"),
|
|
ExchangeStatus.POLL_FAILED,
|
|
),
|
|
],
|
|
)
|
|
def test_expected_transport_failures_have_typed_status(outcome, status):
|
|
assert outcome.status is status
|
|
assert outcome.ok is False
|
|
assert outcome.expected_failure is True
|
|
|
|
|
|
def test_normalize_reply_text_handles_provider_shapes():
|
|
assert normalize_reply_text("plain text") == "plain text"
|
|
assert normalize_reply_text({"content": "content field"}) == "content field"
|
|
assert normalize_reply_text({"msgBody": {"text": "nested text"}}) == "nested text"
|
|
assert normalize_reply_text(None) == ""
|
|
|
|
|
|
def test_success_requires_correlation_text_and_latency():
|
|
with pytest.raises(ValueError, match="correlation_id"):
|
|
ExchangeOutcome(status=ExchangeStatus.SUCCESS, reply_text="ok", latency_ms=1)
|
|
|
|
with pytest.raises(ValueError, match="reply_text"):
|
|
ExchangeOutcome(status=ExchangeStatus.SUCCESS, correlation_id="question-7", latency_ms=1)
|
|
|
|
with pytest.raises(ValueError, match="latency_ms"):
|
|
ExchangeOutcome(status=ExchangeStatus.SUCCESS, correlation_id="question-7", reply_text="ok", latency_ms=-1)
|
|
|
|
|
|
async def test_exchange_runs_send_hook_before_polling_and_returns_success():
|
|
channel = ContractChannel(reply=Reply(question_msg_id="question-7", content={"content": "答复"}))
|
|
|
|
async def on_sent(send_result):
|
|
channel.events.append(f"hook:{send_result.question_msg_id}")
|
|
|
|
outcome = await channel.exchange("问题", on_sent=on_sent)
|
|
|
|
assert outcome.ok is True
|
|
assert outcome.reply_text == "答复"
|
|
assert outcome.correlation_id == "question-7"
|
|
assert channel.events == ["send", "hook:question-7", "poll"]
|
|
|
|
|
|
async def test_exchange_send_failure_skips_hook_and_polling():
|
|
channel = ContractChannel(send_result=SendResult(ok=False, error="offline"))
|
|
|
|
async def on_sent(_send_result):
|
|
raise AssertionError("send hook must not run after a failed send")
|
|
|
|
outcome = await channel.exchange("问题", on_sent=on_sent)
|
|
|
|
assert outcome.status is ExchangeStatus.SEND_FAILED
|
|
assert outcome.reason == "offline"
|
|
assert channel.events == ["send"]
|
|
|
|
|
|
async def test_exchange_hook_failure_skips_polling():
|
|
channel = ContractChannel(reply=Reply(question_msg_id="question-7", content="答复"))
|
|
|
|
async def on_sent(_send_result):
|
|
channel.events.append("hook")
|
|
raise RuntimeError("ledger unavailable")
|
|
|
|
with pytest.raises(RuntimeError, match="ledger unavailable"):
|
|
await channel.exchange("问题", on_sent=on_sent)
|
|
|
|
assert channel.events == ["send", "hook"]
|
|
|
|
|
|
async def test_exchange_distinguishes_timeout_and_poll_failure():
|
|
timeout_channel = ContractChannel()
|
|
timeout = await timeout_channel.exchange("问题")
|
|
assert timeout.status is ExchangeStatus.REPLY_TIMEOUT
|
|
assert timeout.correlation_id == "question-7"
|
|
|
|
failed_channel = ContractChannel(poll_error=ChannelTransportError("upstream unavailable"))
|
|
failed = await failed_channel.exchange("问题")
|
|
assert failed.status is ExchangeStatus.POLL_FAILED
|
|
assert failed.reason == "upstream unavailable"
|
|
|
|
|
|
async def test_exchange_does_not_mask_programming_or_configuration_errors():
|
|
channel = ContractChannel(poll_error=ValueError("invalid adapter configuration"))
|
|
|
|
with pytest.raises(ValueError, match="invalid adapter configuration"):
|
|
await channel.exchange("问题")
|