737c9ab80e
All checks were successful
CI / test (pull_request) Successful in 4m6s
扩展 response_time 规则，支持多维度延迟和吞吐量测量。 - 新增 avg_latency_max_ms：多轮对话平均延迟阈值 - 新增 throughput_min：最低吞吐量（turns/min） - 保持 max_ms 向后兼容（单轮延迟阈值） - RuleResult.details 包含延迟统计（avg/min/max/latencies） - 新增 10 项单元测试（656 tests passed） 注：首字延迟（first token latency）需要流式数据支持，当前 Turn 模型未提供，留待后续增强。 Closes #21
91 lines
3.6 KiB
Python
91 lines
3.6 KiB
Python
"""Response time evaluation rule."""
|
||
|
||
from agenteval.evaluation.rules.base import EvalRule, RuleResult, register_rule
|
||
from agenteval.models import Case, Turn
|
||
|
||
|
||
@register_rule
|
||
class ResponseTimeRule(EvalRule):
|
||
"""Check whether the reply latency is within the configured threshold.
|
||
|
||
Supports multiple metrics:
|
||
- max_ms: threshold for single turn latency (backward compatible)
|
||
- avg_latency_max_ms: threshold for average latency across all turns
|
||
- throughput_min: minimum throughput in turns per minute
|
||
"""
|
||
|
||
name = "response_time"
|
||
|
||
async def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult:
|
||
if not dialog:
|
||
return RuleResult(passed=False, reason="无回复记录")
|
||
|
||
# Collect latencies from all turns
|
||
latencies = [t.latency_ms for t in dialog if t.latency_ms is not None]
|
||
if not latencies:
|
||
return RuleResult(passed=False, reason="无法获取响应时间")
|
||
|
||
# Backward compatible: check last turn against max_ms threshold
|
||
threshold_ms = self.params.get("max_ms")
|
||
if threshold_ms is None:
|
||
threshold_ms = case.expectations.response_time_max_ms
|
||
|
||
results = []
|
||
all_passed = True
|
||
|
||
# 1. Single turn latency check (backward compatible)
|
||
if threshold_ms is not None:
|
||
last_latency = latencies[-1]
|
||
passed = last_latency <= threshold_ms
|
||
if not passed:
|
||
all_passed = False
|
||
results.append(f"最后一轮 {last_latency}ms {'≤' if passed else '>'} {threshold_ms}ms")
|
||
|
||
# 2. Average latency check
|
||
avg_latency_max_ms = self.params.get("avg_latency_max_ms")
|
||
if avg_latency_max_ms is not None:
|
||
avg_latency = sum(latencies) / len(latencies)
|
||
passed = avg_latency <= avg_latency_max_ms
|
||
if not passed:
|
||
all_passed = False
|
||
results.append(f"平均延迟 {avg_latency:.0f}ms {'≤' if passed else '>'} {avg_latency_max_ms}ms")
|
||
|
||
# 3. Throughput check (turns per minute)
|
||
throughput_min = self.params.get("throughput_min")
|
||
if throughput_min is not None and len(dialog) >= 2:
|
||
first_sent = dialog[0].sent_at
|
||
last_received = dialog[-1].received_at
|
||
if first_sent and last_received:
|
||
duration_minutes = (last_received - first_sent).total_seconds() / 60
|
||
if duration_minutes > 0:
|
||
throughput = len(dialog) / duration_minutes
|
||
passed = throughput >= throughput_min
|
||
if not passed:
|
||
all_passed = False
|
||
results.append(f"吞吐量 {throughput:.1f} turns/min {'≥' if passed else '<'} {throughput_min}")
|
||
|
||
# If no metrics configured, just report the last turn latency
|
||
if not results:
|
||
last_latency = latencies[-1]
|
||
return RuleResult(passed=True, score=1.0, reason=f"响应时间 {last_latency}ms")
|
||
|
||
# Calculate score based on average latency
|
||
avg_latency = sum(latencies) / len(latencies)
|
||
if threshold_ms:
|
||
score = 1.0 if avg_latency <= 0 else min(1.0, threshold_ms / avg_latency)
|
||
else:
|
||
score = 1.0
|
||
|
||
verdict = "通过" if all_passed else "未通过"
|
||
return RuleResult(
|
||
passed=all_passed,
|
||
score=score,
|
||
reason=f"响应时间指标 {verdict}：{'; '.join(results)}",
|
||
details={
|
||
"latencies": latencies,
|
||
"avg_latency_ms": avg_latency,
|
||
"min_latency_ms": min(latencies),
|
||
"max_latency_ms": max(latencies),
|
||
},
|
||
)
|