c1a3cdbaa9
All checks were successful
CI / test (pull_request) Successful in 4m5s
- response_time: 仅配置 max_ms 时恢复 v0.3 评分语义（最后一轮评分 + 超限线性惩罚），扩展指标共存时才用均值评分 - safety: 移除 moderation API 的黑名单命中跳过守卫，两层安全检查独立执行、发现均上报 - llm_score: 多维度评分添加 Semaphore 并发上限（5），防止维度数多时无限扇出模型请求
98 lines
4.1 KiB
Python
98 lines
4.1 KiB
Python
"""Response time evaluation rule."""
|
||
|
||
from agenteval.evaluation.rules.base import EvalRule, RuleResult, register_rule
|
||
from agenteval.models import Case, Turn
|
||
|
||
|
||
@register_rule
|
||
class ResponseTimeRule(EvalRule):
|
||
"""Check whether the reply latency is within the configured threshold.
|
||
|
||
Supports multiple metrics:
|
||
- max_ms: threshold for single turn latency (backward compatible)
|
||
- avg_latency_max_ms: threshold for average latency across all turns
|
||
- throughput_min: minimum throughput in turns per minute
|
||
"""
|
||
|
||
name = "response_time"
|
||
|
||
async def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult:
|
||
if not dialog:
|
||
return RuleResult(passed=False, reason="无回复记录")
|
||
|
||
# Collect latencies from all turns
|
||
latencies = [t.latency_ms for t in dialog if t.latency_ms is not None]
|
||
if not latencies:
|
||
return RuleResult(passed=False, reason="无法获取响应时间")
|
||
|
||
# Backward compatible: check last turn against max_ms threshold
|
||
threshold_ms = self.params.get("max_ms")
|
||
if threshold_ms is None:
|
||
threshold_ms = case.expectations.response_time_max_ms
|
||
|
||
results = []
|
||
all_passed = True
|
||
|
||
# 1. Single turn latency check (backward compatible)
|
||
if threshold_ms is not None:
|
||
last_latency = latencies[-1]
|
||
passed = last_latency <= threshold_ms
|
||
if not passed:
|
||
all_passed = False
|
||
results.append(f"最后一轮 {last_latency}ms {'≤' if passed else '>'} {threshold_ms}ms")
|
||
|
||
# 2. Average latency check
|
||
avg_latency_max_ms = self.params.get("avg_latency_max_ms")
|
||
if avg_latency_max_ms is not None:
|
||
avg_latency = sum(latencies) / len(latencies)
|
||
passed = avg_latency <= avg_latency_max_ms
|
||
if not passed:
|
||
all_passed = False
|
||
results.append(f"平均延迟 {avg_latency:.0f}ms {'≤' if passed else '>'} {avg_latency_max_ms}ms")
|
||
|
||
# 3. Throughput check (turns per minute)
|
||
throughput_min = self.params.get("throughput_min")
|
||
if throughput_min is not None and len(dialog) >= 2:
|
||
first_sent = dialog[0].sent_at
|
||
last_received = dialog[-1].received_at
|
||
if first_sent and last_received:
|
||
duration_minutes = (last_received - first_sent).total_seconds() / 60
|
||
if duration_minutes > 0:
|
||
throughput = len(dialog) / duration_minutes
|
||
passed = throughput >= throughput_min
|
||
if not passed:
|
||
all_passed = False
|
||
results.append(f"吞吐量 {throughput:.1f} turns/min {'≥' if passed else '<'} {throughput_min}")
|
||
|
||
# If no metrics configured, just report the last turn latency
|
||
if not results:
|
||
last_latency = latencies[-1]
|
||
return RuleResult(passed=True, score=1.0, reason=f"响应时间 {last_latency}ms")
|
||
|
||
avg_latency = sum(latencies) / len(latencies)
|
||
has_extended_metrics = avg_latency_max_ms is not None or throughput_min is not None
|
||
if threshold_ms is not None and not has_extended_metrics:
|
||
# 仅 max_ms：保持 v0.3 原语义——基于最后一轮评分，超限有惩罚
|
||
last_latency = latencies[-1]
|
||
if last_latency > threshold_ms:
|
||
score = max(0.0, 1.0 - (last_latency - threshold_ms) / threshold_ms)
|
||
else:
|
||
score = 1.0 if last_latency <= 0 else min(1.0, threshold_ms / last_latency)
|
||
elif threshold_ms is not None:
|
||
score = 1.0 if avg_latency <= 0 else min(1.0, threshold_ms / avg_latency)
|
||
else:
|
||
score = 1.0
|
||
|
||
verdict = "通过" if all_passed else "未通过"
|
||
return RuleResult(
|
||
passed=all_passed,
|
||
score=score,
|
||
reason=f"响应时间指标 {verdict}：{'; '.join(results)}",
|
||
details={
|
||
"latencies": latencies,
|
||
"avg_latency_ms": avg_latency,
|
||
"min_latency_ms": min(latencies),
|
||
"max_latency_ms": max(latencies),
|
||
},
|
||
)
|