feat(response_time): 支持平均延迟和吞吐量指标
All checks were successful
CI / test (pull_request) Successful in 4m6s
All checks were successful
CI / test (pull_request) Successful in 4m6s
扩展 response_time 规则,支持多维度延迟和吞吐量测量。 - 新增 avg_latency_max_ms:多轮对话平均延迟阈值 - 新增 throughput_min:最低吞吐量(turns/min) - 保持 max_ms 向后兼容(单轮延迟阈值) - RuleResult.details 包含延迟统计(avg/min/max/latencies) - 新增 10 项单元测试(656 tests passed) 注:首字延迟(first token latency)需要流式数据支持,当前 Turn 模型未提供,留待后续增强。 Closes #21
This commit is contained in:
parent
2f08e7bf06
commit
737c9ab80e
@ -6,7 +6,13 @@ from agenteval.models import Case, Turn
|
||||
|
||||
@register_rule
|
||||
class ResponseTimeRule(EvalRule):
|
||||
"""Check whether the reply latency is within the configured threshold."""
|
||||
"""Check whether the reply latency is within the configured threshold.
|
||||
|
||||
Supports multiple metrics:
|
||||
- max_ms: threshold for single turn latency (backward compatible)
|
||||
- avg_latency_max_ms: threshold for average latency across all turns
|
||||
- throughput_min: minimum throughput in turns per minute
|
||||
"""
|
||||
|
||||
name = "response_time"
|
||||
|
||||
@ -14,23 +20,71 @@ class ResponseTimeRule(EvalRule):
|
||||
if not dialog:
|
||||
return RuleResult(passed=False, reason="无回复记录")
|
||||
|
||||
# Collect latencies from all turns
|
||||
latencies = [t.latency_ms for t in dialog if t.latency_ms is not None]
|
||||
if not latencies:
|
||||
return RuleResult(passed=False, reason="无法获取响应时间")
|
||||
|
||||
# Backward compatible: check last turn against max_ms threshold
|
||||
threshold_ms = self.params.get("max_ms")
|
||||
if threshold_ms is None:
|
||||
threshold_ms = case.expectations.response_time_max_ms
|
||||
if threshold_ms is None:
|
||||
return RuleResult(passed=True, reason="未配置响应时间阈值")
|
||||
|
||||
last_turn = dialog[-1]
|
||||
latency = last_turn.latency_ms
|
||||
if latency is None:
|
||||
return RuleResult(passed=False, reason="无法获取响应时间")
|
||||
results = []
|
||||
all_passed = True
|
||||
|
||||
if latency > threshold_ms:
|
||||
# 1. Single turn latency check (backward compatible)
|
||||
if threshold_ms is not None:
|
||||
last_latency = latencies[-1]
|
||||
passed = last_latency <= threshold_ms
|
||||
if not passed:
|
||||
all_passed = False
|
||||
results.append(f"最后一轮 {last_latency}ms {'≤' if passed else '>'} {threshold_ms}ms")
|
||||
|
||||
# 2. Average latency check
|
||||
avg_latency_max_ms = self.params.get("avg_latency_max_ms")
|
||||
if avg_latency_max_ms is not None:
|
||||
avg_latency = sum(latencies) / len(latencies)
|
||||
passed = avg_latency <= avg_latency_max_ms
|
||||
if not passed:
|
||||
all_passed = False
|
||||
results.append(f"平均延迟 {avg_latency:.0f}ms {'≤' if passed else '>'} {avg_latency_max_ms}ms")
|
||||
|
||||
# 3. Throughput check (turns per minute)
|
||||
throughput_min = self.params.get("throughput_min")
|
||||
if throughput_min is not None and len(dialog) >= 2:
|
||||
first_sent = dialog[0].sent_at
|
||||
last_received = dialog[-1].received_at
|
||||
if first_sent and last_received:
|
||||
duration_minutes = (last_received - first_sent).total_seconds() / 60
|
||||
if duration_minutes > 0:
|
||||
throughput = len(dialog) / duration_minutes
|
||||
passed = throughput >= throughput_min
|
||||
if not passed:
|
||||
all_passed = False
|
||||
results.append(f"吞吐量 {throughput:.1f} turns/min {'≥' if passed else '<'} {throughput_min}")
|
||||
|
||||
# If no metrics configured, just report the last turn latency
|
||||
if not results:
|
||||
last_latency = latencies[-1]
|
||||
return RuleResult(passed=True, score=1.0, reason=f"响应时间 {last_latency}ms")
|
||||
|
||||
# Calculate score based on average latency
|
||||
avg_latency = sum(latencies) / len(latencies)
|
||||
if threshold_ms:
|
||||
score = 1.0 if avg_latency <= 0 else min(1.0, threshold_ms / avg_latency)
|
||||
else:
|
||||
score = 1.0
|
||||
|
||||
verdict = "通过" if all_passed else "未通过"
|
||||
return RuleResult(
|
||||
passed=False,
|
||||
score=max(0.0, 1.0 - (latency - threshold_ms) / threshold_ms),
|
||||
reason=f"响应时间 {latency}ms 超过阈值 {threshold_ms}ms",
|
||||
passed=all_passed,
|
||||
score=score,
|
||||
reason=f"响应时间指标 {verdict}:{'; '.join(results)}",
|
||||
details={
|
||||
"latencies": latencies,
|
||||
"avg_latency_ms": avg_latency,
|
||||
"min_latency_ms": min(latencies),
|
||||
"max_latency_ms": max(latencies),
|
||||
},
|
||||
)
|
||||
|
||||
score = 1.0 if latency <= 0 else min(1.0, threshold_ms / latency)
|
||||
return RuleResult(passed=True, score=score, reason=f"响应时间 {latency}ms 通过")
|
||||
|
||||
164
tests/unit/test_response_time_extended.py
Normal file
164
tests/unit/test_response_time_extended.py
Normal file
@ -0,0 +1,164 @@
|
||||
"""Tests for extended response_time rule with multiple metrics."""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from agenteval.evaluation.rules.base import get_rule
|
||||
from agenteval.models import Case, Expectation, Turn
|
||||
|
||||
|
||||
def _make_turn(round_index: int, latency_ms: int, sent_at: datetime | None = None, received_at: datetime | None = None) -> Turn:
|
||||
"""Helper to create a Turn with latency."""
|
||||
return Turn(
|
||||
id=f"t-{round_index}",
|
||||
run_id="r1",
|
||||
case_id="c1",
|
||||
round_index=round_index,
|
||||
latency_ms=latency_ms,
|
||||
sent_at=sent_at,
|
||||
received_at=received_at,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_time_backward_compatible():
|
||||
"""Single turn with max_ms threshold should work as before."""
|
||||
rule = get_rule("response_time", {"max_ms": 5000})
|
||||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||||
dialog = [_make_turn(1, 3000)]
|
||||
result = await rule.evaluate(case, dialog)
|
||||
assert result.passed is True
|
||||
assert "3000ms" in result.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_time_exceeds_threshold():
|
||||
"""Single turn exceeding threshold should fail."""
|
||||
rule = get_rule("response_time", {"max_ms": 2000})
|
||||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||||
dialog = [_make_turn(1, 3000)]
|
||||
result = await rule.evaluate(case, dialog)
|
||||
assert result.passed is False
|
||||
assert "未通过" in result.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_time_avg_latency():
|
||||
"""Average latency check across multiple turns."""
|
||||
rule = get_rule("response_time", {"avg_latency_max_ms": 3000})
|
||||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||||
dialog = [
|
||||
_make_turn(1, 2000),
|
||||
_make_turn(2, 4000),
|
||||
]
|
||||
result = await rule.evaluate(case, dialog)
|
||||
assert result.passed is True # avg = 3000, threshold = 3000
|
||||
assert "平均延迟" in result.reason
|
||||
assert result.details is not None
|
||||
assert result.details["avg_latency_ms"] == 3000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_time_avg_latency_exceeded():
|
||||
"""Average latency exceeding threshold should fail."""
|
||||
rule = get_rule("response_time", {"avg_latency_max_ms": 2000})
|
||||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||||
dialog = [
|
||||
_make_turn(1, 3000),
|
||||
_make_turn(2, 4000),
|
||||
]
|
||||
result = await rule.evaluate(case, dialog)
|
||||
assert result.passed is False # avg = 3500 > 2000
|
||||
assert "未通过" in result.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_time_throughput():
|
||||
"""Throughput check (turns per minute)."""
|
||||
rule = get_rule("response_time", {"throughput_min": 10})
|
||||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||||
base_time = datetime(2026, 1, 1, 12, 0, 0)
|
||||
dialog = [
|
||||
_make_turn(1, 1000, sent_at=base_time, received_at=base_time + timedelta(seconds=1)),
|
||||
_make_turn(2, 1000, sent_at=base_time + timedelta(seconds=2), received_at=base_time + timedelta(seconds=3)),
|
||||
_make_turn(3, 1000, sent_at=base_time + timedelta(seconds=4), received_at=base_time + timedelta(seconds=5)),
|
||||
]
|
||||
# 3 turns in 5 seconds = 36 turns/min
|
||||
result = await rule.evaluate(case, dialog)
|
||||
assert result.passed is True
|
||||
assert "吞吐量" in result.reason
|
||||
assert "turns/min" in result.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_time_throughput_exceeded():
|
||||
"""Throughput below threshold should fail."""
|
||||
rule = get_rule("response_time", {"throughput_min": 100})
|
||||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||||
base_time = datetime(2026, 1, 1, 12, 0, 0)
|
||||
dialog = [
|
||||
_make_turn(1, 1000, sent_at=base_time, received_at=base_time + timedelta(seconds=10)),
|
||||
_make_turn(2, 1000, sent_at=base_time + timedelta(seconds=20), received_at=base_time + timedelta(seconds=30)),
|
||||
]
|
||||
# 2 turns in 30 seconds = 4 turns/min < 100
|
||||
result = await rule.evaluate(case, dialog)
|
||||
assert result.passed is False
|
||||
assert "未通过" in result.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_time_multiple_metrics():
|
||||
"""Multiple metrics can be checked together."""
|
||||
rule = get_rule("response_time", {
|
||||
"max_ms": 5000,
|
||||
"avg_latency_max_ms": 4000,
|
||||
})
|
||||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||||
dialog = [
|
||||
_make_turn(1, 3000),
|
||||
_make_turn(2, 4500),
|
||||
]
|
||||
result = await rule.evaluate(case, dialog)
|
||||
assert result.passed is True
|
||||
assert "最后一轮" in result.reason
|
||||
assert "平均延迟" in result.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_time_details_field():
|
||||
"""Result should include details with latency statistics."""
|
||||
rule = get_rule("response_time", {"max_ms": 5000})
|
||||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||||
dialog = [
|
||||
_make_turn(1, 2000),
|
||||
_make_turn(2, 3000),
|
||||
_make_turn(3, 4000),
|
||||
]
|
||||
result = await rule.evaluate(case, dialog)
|
||||
assert result.details is not None
|
||||
assert result.details["latencies"] == [2000, 3000, 4000]
|
||||
assert result.details["avg_latency_ms"] == 3000
|
||||
assert result.details["min_latency_ms"] == 2000
|
||||
assert result.details["max_latency_ms"] == 4000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_time_no_threshold():
|
||||
"""No threshold configured should pass with basic info."""
|
||||
rule = get_rule("response_time", {})
|
||||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||||
dialog = [_make_turn(1, 3000)]
|
||||
result = await rule.evaluate(case, dialog)
|
||||
assert result.passed is True
|
||||
assert "3000ms" in result.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_time_empty_dialog():
|
||||
"""Empty dialog should fail."""
|
||||
rule = get_rule("response_time", {"max_ms": 5000})
|
||||
case = Case(id="c1", messages=["hello"], expectations=Expectation())
|
||||
result = await rule.evaluate(case, [])
|
||||
assert result.passed is False
|
||||
assert "无回复记录" in result.reason
|
||||
Loading…
Reference in New Issue
Block a user