"""Tests for extended response_time rule with multiple metrics.""" from datetime import datetime, timedelta import pytest from agenteval.evaluation.rules.base import get_rule from agenteval.models import Case, Expectation, Turn def _make_turn(round_index: int, latency_ms: int, sent_at: datetime | None = None, received_at: datetime | None = None) -> Turn: """Helper to create a Turn with latency.""" return Turn( id=f"t-{round_index}", run_id="r1", case_id="c1", round_index=round_index, latency_ms=latency_ms, sent_at=sent_at, received_at=received_at, ) @pytest.mark.asyncio async def test_response_time_backward_compatible(): """Single turn with max_ms threshold should work as before.""" rule = get_rule("response_time", {"max_ms": 5000}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) dialog = [_make_turn(1, 3000)] result = await rule.evaluate(case, dialog) assert result.passed is True assert "3000ms" in result.reason @pytest.mark.asyncio async def test_response_time_exceeds_threshold(): """Single turn exceeding threshold should fail.""" rule = get_rule("response_time", {"max_ms": 2000}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) dialog = [_make_turn(1, 3000)] result = await rule.evaluate(case, dialog) assert result.passed is False assert "未通过" in result.reason @pytest.mark.asyncio async def test_response_time_avg_latency(): """Average latency check across multiple turns.""" rule = get_rule("response_time", {"avg_latency_max_ms": 3000}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) dialog = [ _make_turn(1, 2000), _make_turn(2, 4000), ] result = await rule.evaluate(case, dialog) assert result.passed is True # avg = 3000, threshold = 3000 assert "平均延迟" in result.reason assert result.details is not None assert result.details["avg_latency_ms"] == 3000 @pytest.mark.asyncio async def test_response_time_avg_latency_exceeded(): """Average latency exceeding threshold should fail.""" rule = get_rule("response_time", {"avg_latency_max_ms": 2000}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) dialog = [ _make_turn(1, 3000), _make_turn(2, 4000), ] result = await rule.evaluate(case, dialog) assert result.passed is False # avg = 3500 > 2000 assert "未通过" in result.reason @pytest.mark.asyncio async def test_response_time_throughput(): """Throughput check (turns per minute).""" rule = get_rule("response_time", {"throughput_min": 10}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) base_time = datetime(2026, 1, 1, 12, 0, 0) dialog = [ _make_turn(1, 1000, sent_at=base_time, received_at=base_time + timedelta(seconds=1)), _make_turn(2, 1000, sent_at=base_time + timedelta(seconds=2), received_at=base_time + timedelta(seconds=3)), _make_turn(3, 1000, sent_at=base_time + timedelta(seconds=4), received_at=base_time + timedelta(seconds=5)), ] # 3 turns in 5 seconds = 36 turns/min result = await rule.evaluate(case, dialog) assert result.passed is True assert "吞吐量" in result.reason assert "turns/min" in result.reason @pytest.mark.asyncio async def test_response_time_throughput_exceeded(): """Throughput below threshold should fail.""" rule = get_rule("response_time", {"throughput_min": 100}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) base_time = datetime(2026, 1, 1, 12, 0, 0) dialog = [ _make_turn(1, 1000, sent_at=base_time, received_at=base_time + timedelta(seconds=10)), _make_turn(2, 1000, sent_at=base_time + timedelta(seconds=20), received_at=base_time + timedelta(seconds=30)), ] # 2 turns in 30 seconds = 4 turns/min < 100 result = await rule.evaluate(case, dialog) assert result.passed is False assert "未通过" in result.reason @pytest.mark.asyncio async def test_response_time_multiple_metrics(): """Multiple metrics can be checked together.""" rule = get_rule("response_time", { "max_ms": 5000, "avg_latency_max_ms": 4000, }) case = Case(id="c1", messages=["hello"], expectations=Expectation()) dialog = [ _make_turn(1, 3000), _make_turn(2, 4500), ] result = await rule.evaluate(case, dialog) assert result.passed is True assert "最后一轮" in result.reason assert "平均延迟" in result.reason @pytest.mark.asyncio async def test_response_time_details_field(): """Result should include details with latency statistics.""" rule = get_rule("response_time", {"max_ms": 5000}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) dialog = [ _make_turn(1, 2000), _make_turn(2, 3000), _make_turn(3, 4000), ] result = await rule.evaluate(case, dialog) assert result.details is not None assert result.details["latencies"] == [2000, 3000, 4000] assert result.details["avg_latency_ms"] == 3000 assert result.details["min_latency_ms"] == 2000 assert result.details["max_latency_ms"] == 4000 @pytest.mark.asyncio async def test_response_time_no_threshold(): """No threshold configured should pass with basic info.""" rule = get_rule("response_time", {}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) dialog = [_make_turn(1, 3000)] result = await rule.evaluate(case, dialog) assert result.passed is True assert "3000ms" in result.reason @pytest.mark.asyncio async def test_response_time_empty_dialog(): """Empty dialog should fail.""" rule = get_rule("response_time", {"max_ms": 5000}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) result = await rule.evaluate(case, []) assert result.passed is False assert "无回复记录" in result.reason @pytest.mark.asyncio async def test_response_time_max_ms_only_exceed_penalty(): """仅 max_ms 时保持 v0.3 语义:超限有线性惩罚(score = 1 - 超出/阈值)。""" rule = get_rule("response_time", {"max_ms": 2000}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) dialog = [_make_turn(1, 3000)] result = await rule.evaluate(case, dialog) assert result.passed is False assert result.score == pytest.approx(0.5) # 1 - (3000-2000)/2000 @pytest.mark.asyncio async def test_response_time_max_ms_only_last_turn_decides(): """仅 max_ms 时保持 v0.3 语义:最后一轮决定评分,即使平均更快。""" rule = get_rule("response_time", {"max_ms": 6000}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) # 最后一轮 5000ms 未超限 → score 1.0(平均 3000ms 不影响) dialog = [_make_turn(1, 1000), _make_turn(2, 5000)] result = await rule.evaluate(case, dialog) assert result.passed is True assert result.score == pytest.approx(1.0) @pytest.mark.asyncio async def test_response_time_mixed_metrics_uses_average_score(): """配置了扩展指标时 score 基于平均延迟。""" rule = get_rule("response_time", {"max_ms": 6000, "avg_latency_max_ms": 4000}) case = Case(id="c1", messages=["hello"], expectations=Expectation()) dialog = [_make_turn(1, 2000), _make_turn(2, 4000)] result = await rule.evaluate(case, dialog) assert result.passed is True assert result.score == pytest.approx(1.0) # avg=3000 <= 6000 → min(1, 6000/3000)=1