"""Response time evaluation rule.""" from agenteval.evaluation.rules.base import EvalRule, RuleResult, register_rule from agenteval.models import Case, Turn @register_rule class ResponseTimeRule(EvalRule): """Check whether the reply latency is within the configured threshold. Supports multiple metrics: - max_ms: threshold for single turn latency (backward compatible) - avg_latency_max_ms: threshold for average latency across all turns - throughput_min: minimum throughput in turns per minute """ name = "response_time" async def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult: if not dialog: return RuleResult(passed=False, reason="无回复记录") # Collect latencies from all turns latencies = [t.latency_ms for t in dialog if t.latency_ms is not None] if not latencies: return RuleResult(passed=False, reason="无法获取响应时间") # Backward compatible: check last turn against max_ms threshold threshold_ms = self.params.get("max_ms") if threshold_ms is None: threshold_ms = case.expectations.response_time_max_ms results = [] all_passed = True # 1. Single turn latency check (backward compatible) if threshold_ms is not None: last_latency = latencies[-1] passed = last_latency <= threshold_ms if not passed: all_passed = False results.append(f"最后一轮 {last_latency}ms {'≤' if passed else '>'} {threshold_ms}ms") # 2. Average latency check avg_latency_max_ms = self.params.get("avg_latency_max_ms") if avg_latency_max_ms is not None: avg_latency = sum(latencies) / len(latencies) passed = avg_latency <= avg_latency_max_ms if not passed: all_passed = False results.append(f"平均延迟 {avg_latency:.0f}ms {'≤' if passed else '>'} {avg_latency_max_ms}ms") # 3. Throughput check (turns per minute) throughput_min = self.params.get("throughput_min") if throughput_min is not None and len(dialog) >= 2: first_sent = dialog[0].sent_at last_received = dialog[-1].received_at if first_sent and last_received: duration_minutes = (last_received - first_sent).total_seconds() / 60 if duration_minutes > 0: throughput = len(dialog) / duration_minutes passed = throughput >= throughput_min if not passed: all_passed = False results.append(f"吞吐量 {throughput:.1f} turns/min {'≥' if passed else '<'} {throughput_min}") # If no metrics configured, just report the last turn latency if not results: last_latency = latencies[-1] return RuleResult(passed=True, score=1.0, reason=f"响应时间 {last_latency}ms") avg_latency = sum(latencies) / len(latencies) has_extended_metrics = avg_latency_max_ms is not None or throughput_min is not None if threshold_ms is not None and not has_extended_metrics: # 仅 max_ms:保持 v0.3 原语义——基于最后一轮评分,超限有惩罚 last_latency = latencies[-1] if last_latency > threshold_ms: score = max(0.0, 1.0 - (last_latency - threshold_ms) / threshold_ms) else: score = 1.0 if last_latency <= 0 else min(1.0, threshold_ms / last_latency) elif threshold_ms is not None: score = 1.0 if avg_latency <= 0 else min(1.0, threshold_ms / avg_latency) else: score = 1.0 verdict = "通过" if all_passed else "未通过" return RuleResult( passed=all_passed, score=score, reason=f"响应时间指标 {verdict}:{'; '.join(results)}", details={ "latencies": latencies, "avg_latency_ms": avg_latency, "min_latency_ms": min(latencies), "max_latency_ms": max(latencies), }, )