c1a3cdbaa9
All checks were successful
CI / test (pull_request) Successful in 4m5s
- response_time: 仅配置 max_ms 时恢复 v0.3 评分语义（最后一轮评分 + 超限线性惩罚），扩展指标共存时才用均值评分 - safety: 移除 moderation API 的黑名单命中跳过守卫，两层安全检查独立执行、发现均上报 - llm_score: 多维度评分添加 Semaphore 并发上限（5），防止维度数多时无限扇出模型请求
199 lines
7.5 KiB
Python
199 lines
7.5 KiB
Python
"""Tests for extended response_time rule with multiple metrics."""
|
||
|
||
from datetime import datetime, timedelta
|
||
|
||
import pytest
|
||
|
||
from agenteval.evaluation.rules.base import get_rule
|
||
from agenteval.models import Case, Expectation, Turn
|
||
|
||
|
||
def _make_turn(round_index: int, latency_ms: int, sent_at: datetime | None = None, received_at: datetime | None = None) -> Turn:
|
||
"""Helper to create a Turn with latency."""
|
||
return Turn(
|
||
id=f"t-{round_index}",
|
||
run_id="r1",
|
||
case_id="c1",
|
||
round_index=round_index,
|
||
latency_ms=latency_ms,
|
||
sent_at=sent_at,
|
||
received_at=received_at,
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_backward_compatible():
|
||
"""Single turn with max_ms threshold should work as before."""
|
||
rule = get_rule("response_time", {"max_ms": 5000})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
dialog = [_make_turn(1, 3000)]
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is True
|
||
assert "3000ms" in result.reason
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_exceeds_threshold():
|
||
"""Single turn exceeding threshold should fail."""
|
||
rule = get_rule("response_time", {"max_ms": 2000})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
dialog = [_make_turn(1, 3000)]
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is False
|
||
assert "未通过" in result.reason
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_avg_latency():
|
||
"""Average latency check across multiple turns."""
|
||
rule = get_rule("response_time", {"avg_latency_max_ms": 3000})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
dialog = [
|
||
_make_turn(1, 2000),
|
||
_make_turn(2, 4000),
|
||
]
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is True # avg = 3000, threshold = 3000
|
||
assert "平均延迟" in result.reason
|
||
assert result.details is not None
|
||
assert result.details["avg_latency_ms"] == 3000
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_avg_latency_exceeded():
|
||
"""Average latency exceeding threshold should fail."""
|
||
rule = get_rule("response_time", {"avg_latency_max_ms": 2000})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
dialog = [
|
||
_make_turn(1, 3000),
|
||
_make_turn(2, 4000),
|
||
]
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is False # avg = 3500 > 2000
|
||
assert "未通过" in result.reason
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_throughput():
|
||
"""Throughput check (turns per minute)."""
|
||
rule = get_rule("response_time", {"throughput_min": 10})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
base_time = datetime(2026, 1, 1, 12, 0, 0)
|
||
dialog = [
|
||
_make_turn(1, 1000, sent_at=base_time, received_at=base_time + timedelta(seconds=1)),
|
||
_make_turn(2, 1000, sent_at=base_time + timedelta(seconds=2), received_at=base_time + timedelta(seconds=3)),
|
||
_make_turn(3, 1000, sent_at=base_time + timedelta(seconds=4), received_at=base_time + timedelta(seconds=5)),
|
||
]
|
||
# 3 turns in 5 seconds = 36 turns/min
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is True
|
||
assert "吞吐量" in result.reason
|
||
assert "turns/min" in result.reason
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_throughput_exceeded():
|
||
"""Throughput below threshold should fail."""
|
||
rule = get_rule("response_time", {"throughput_min": 100})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
base_time = datetime(2026, 1, 1, 12, 0, 0)
|
||
dialog = [
|
||
_make_turn(1, 1000, sent_at=base_time, received_at=base_time + timedelta(seconds=10)),
|
||
_make_turn(2, 1000, sent_at=base_time + timedelta(seconds=20), received_at=base_time + timedelta(seconds=30)),
|
||
]
|
||
# 2 turns in 30 seconds = 4 turns/min < 100
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is False
|
||
assert "未通过" in result.reason
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_multiple_metrics():
|
||
"""Multiple metrics can be checked together."""
|
||
rule = get_rule("response_time", {
|
||
"max_ms": 5000,
|
||
"avg_latency_max_ms": 4000,
|
||
})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
dialog = [
|
||
_make_turn(1, 3000),
|
||
_make_turn(2, 4500),
|
||
]
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is True
|
||
assert "最后一轮" in result.reason
|
||
assert "平均延迟" in result.reason
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_details_field():
|
||
"""Result should include details with latency statistics."""
|
||
rule = get_rule("response_time", {"max_ms": 5000})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
dialog = [
|
||
_make_turn(1, 2000),
|
||
_make_turn(2, 3000),
|
||
_make_turn(3, 4000),
|
||
]
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.details is not None
|
||
assert result.details["latencies"] == [2000, 3000, 4000]
|
||
assert result.details["avg_latency_ms"] == 3000
|
||
assert result.details["min_latency_ms"] == 2000
|
||
assert result.details["max_latency_ms"] == 4000
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_no_threshold():
|
||
"""No threshold configured should pass with basic info."""
|
||
rule = get_rule("response_time", {})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
dialog = [_make_turn(1, 3000)]
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is True
|
||
assert "3000ms" in result.reason
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_empty_dialog():
|
||
"""Empty dialog should fail."""
|
||
rule = get_rule("response_time", {"max_ms": 5000})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
result = await rule.evaluate(case, [])
|
||
assert result.passed is False
|
||
assert "无回复记录" in result.reason
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_max_ms_only_exceed_penalty():
|
||
"""仅 max_ms 时保持 v0.3 语义：超限有线性惩罚（score = 1 - 超出/阈值）。"""
|
||
rule = get_rule("response_time", {"max_ms": 2000})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
dialog = [_make_turn(1, 3000)]
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is False
|
||
assert result.score == pytest.approx(0.5) # 1 - (3000-2000)/2000
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_max_ms_only_last_turn_decides():
|
||
"""仅 max_ms 时保持 v0.3 语义：最后一轮决定评分，即使平均更快。"""
|
||
rule = get_rule("response_time", {"max_ms": 6000})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
# 最后一轮 5000ms 未超限 → score 1.0（平均 3000ms 不影响）
|
||
dialog = [_make_turn(1, 1000), _make_turn(2, 5000)]
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is True
|
||
assert result.score == pytest.approx(1.0)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_response_time_mixed_metrics_uses_average_score():
|
||
"""配置了扩展指标时 score 基于平均延迟。"""
|
||
rule = get_rule("response_time", {"max_ms": 6000, "avg_latency_max_ms": 4000})
|
||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||
dialog = [_make_turn(1, 2000), _make_turn(2, 4000)]
|
||
result = await rule.evaluate(case, dialog)
|
||
assert result.passed is True
|
||
assert result.score == pytest.approx(1.0) # avg=3000 <= 6000 → min(1, 6000/3000)=1
|