From 737c9ab80e7da225d0fe5065523497607332b190 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Tue, 25 Aug 2026 15:03:54 +0800 Subject: [PATCH] =?UTF-8?q?feat(response=5Ftime):=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=B9=B3=E5=9D=87=E5=BB=B6=E8=BF=9F=E5=92=8C=E5=90=9E=E5=90=90?= =?UTF-8?q?=E9=87=8F=E6=8C=87=E6=A0=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 扩展 response_time 规则，支持多维度延迟和吞吐量测量。 - 新增 avg_latency_max_ms：多轮对话平均延迟阈值 - 新增 throughput_min：最低吞吐量（turns/min） - 保持 max_ms 向后兼容（单轮延迟阈值） - RuleResult.details 包含延迟统计（avg/min/max/latencies） - 新增 10 项单元测试（656 tests passed） 注：首字延迟（first token latency）需要流式数据支持，当前 Turn 模型未提供，留待后续增强。 Closes #21 --- .../evaluation/rules/response_time.py | 84 +++++++-- tests/unit/test_response_time_extended.py | 164 ++++++++++++++++++ 2 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 tests/unit/test_response_time_extended.py diff --git a/backend/agenteval/evaluation/rules/response_time.py b/backend/agenteval/evaluation/rules/response_time.py index acbbea8..652a8f2 100644 --- a/backend/agenteval/evaluation/rules/response_time.py +++ b/backend/agenteval/evaluation/rules/response_time.py @@ -6,7 +6,13 @@ from agenteval.models import Case, Turn @register_rule class ResponseTimeRule(EvalRule): - """Check whether the reply latency is within the configured threshold.""" + """Check whether the reply latency is within the configured threshold. + + Supports multiple metrics: + - max_ms: threshold for single turn latency (backward compatible) + - avg_latency_max_ms: threshold for average latency across all turns + - throughput_min: minimum throughput in turns per minute + """ name = "response_time" @@ -14,23 +20,71 @@ class ResponseTimeRule(EvalRule): if not dialog: return RuleResult(passed=False, reason="无回复记录") + # Collect latencies from all turns + latencies = [t.latency_ms for t in dialog if t.latency_ms is not None] + if not latencies: + return RuleResult(passed=False, reason="无法获取响应时间") + + # Backward compatible: check last turn against max_ms threshold threshold_ms = self.params.get("max_ms") if threshold_ms is None: threshold_ms = case.expectations.response_time_max_ms - if threshold_ms is None: - return RuleResult(passed=True, reason="未配置响应时间阈值") - last_turn = dialog[-1] - latency = last_turn.latency_ms - if latency is None: - return RuleResult(passed=False, reason="无法获取响应时间") + results = [] + all_passed = True - if latency > threshold_ms: - return RuleResult( - passed=False, - score=max(0.0, 1.0 - (latency - threshold_ms) / threshold_ms), - reason=f"响应时间 {latency}ms 超过阈值 {threshold_ms}ms", - ) + # 1. Single turn latency check (backward compatible) + if threshold_ms is not None: + last_latency = latencies[-1] + passed = last_latency <= threshold_ms + if not passed: + all_passed = False + results.append(f"最后一轮 {last_latency}ms {'≤' if passed else '>'} {threshold_ms}ms") - score = 1.0 if latency <= 0 else min(1.0, threshold_ms / latency) - return RuleResult(passed=True, score=score, reason=f"响应时间 {latency}ms 通过") + # 2. Average latency check + avg_latency_max_ms = self.params.get("avg_latency_max_ms") + if avg_latency_max_ms is not None: + avg_latency = sum(latencies) / len(latencies) + passed = avg_latency <= avg_latency_max_ms + if not passed: + all_passed = False + results.append(f"平均延迟 {avg_latency:.0f}ms {'≤' if passed else '>'} {avg_latency_max_ms}ms") + + # 3. Throughput check (turns per minute) + throughput_min = self.params.get("throughput_min") + if throughput_min is not None and len(dialog) >= 2: + first_sent = dialog[0].sent_at + last_received = dialog[-1].received_at + if first_sent and last_received: + duration_minutes = (last_received - first_sent).total_seconds() / 60 + if duration_minutes > 0: + throughput = len(dialog) / duration_minutes + passed = throughput >= throughput_min + if not passed: + all_passed = False + results.append(f"吞吐量 {throughput:.1f} turns/min {'≥' if passed else '<'} {throughput_min}") + + # If no metrics configured, just report the last turn latency + if not results: + last_latency = latencies[-1] + return RuleResult(passed=True, score=1.0, reason=f"响应时间 {last_latency}ms") + + # Calculate score based on average latency + avg_latency = sum(latencies) / len(latencies) + if threshold_ms: + score = 1.0 if avg_latency <= 0 else min(1.0, threshold_ms / avg_latency) + else: + score = 1.0 + + verdict = "通过" if all_passed else "未通过" + return RuleResult( + passed=all_passed, + score=score, + reason=f"响应时间指标 {verdict}：{'; '.join(results)}", + details={ + "latencies": latencies, + "avg_latency_ms": avg_latency, + "min_latency_ms": min(latencies), + "max_latency_ms": max(latencies), + }, + ) diff --git a/tests/unit/test_response_time_extended.py b/tests/unit/test_response_time_extended.py new file mode 100644 index 0000000..a20748c --- /dev/null +++ b/tests/unit/test_response_time_extended.py @@ -0,0 +1,164 @@ +"""Tests for extended response_time rule with multiple metrics.""" + +from datetime import datetime, timedelta + +import pytest + +from agenteval.evaluation.rules.base import get_rule +from agenteval.models import Case, Expectation, Turn + + +def _make_turn(round_index: int, latency_ms: int, sent_at: datetime | None = None, received_at: datetime | None = None) -> Turn: + """Helper to create a Turn with latency.""" + return Turn( + id=f"t-{round_index}", + run_id="r1", + case_id="c1", + round_index=round_index, + latency_ms=latency_ms, + sent_at=sent_at, + received_at=received_at, + ) + + +@pytest.mark.asyncio +async def test_response_time_backward_compatible(): + """Single turn with max_ms threshold should work as before.""" + rule = get_rule("response_time", {"max_ms": 5000}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + dialog = [_make_turn(1, 3000)] + result = await rule.evaluate(case, dialog) + assert result.passed is True + assert "3000ms" in result.reason + + +@pytest.mark.asyncio +async def test_response_time_exceeds_threshold(): + """Single turn exceeding threshold should fail.""" + rule = get_rule("response_time", {"max_ms": 2000}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + dialog = [_make_turn(1, 3000)] + result = await rule.evaluate(case, dialog) + assert result.passed is False + assert "未通过" in result.reason + + +@pytest.mark.asyncio +async def test_response_time_avg_latency(): + """Average latency check across multiple turns.""" + rule = get_rule("response_time", {"avg_latency_max_ms": 3000}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + dialog = [ + _make_turn(1, 2000), + _make_turn(2, 4000), + ] + result = await rule.evaluate(case, dialog) + assert result.passed is True # avg = 3000, threshold = 3000 + assert "平均延迟" in result.reason + assert result.details is not None + assert result.details["avg_latency_ms"] == 3000 + + +@pytest.mark.asyncio +async def test_response_time_avg_latency_exceeded(): + """Average latency exceeding threshold should fail.""" + rule = get_rule("response_time", {"avg_latency_max_ms": 2000}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + dialog = [ + _make_turn(1, 3000), + _make_turn(2, 4000), + ] + result = await rule.evaluate(case, dialog) + assert result.passed is False # avg = 3500 > 2000 + assert "未通过" in result.reason + + +@pytest.mark.asyncio +async def test_response_time_throughput(): + """Throughput check (turns per minute).""" + rule = get_rule("response_time", {"throughput_min": 10}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + base_time = datetime(2026, 1, 1, 12, 0, 0) + dialog = [ + _make_turn(1, 1000, sent_at=base_time, received_at=base_time + timedelta(seconds=1)), + _make_turn(2, 1000, sent_at=base_time + timedelta(seconds=2), received_at=base_time + timedelta(seconds=3)), + _make_turn(3, 1000, sent_at=base_time + timedelta(seconds=4), received_at=base_time + timedelta(seconds=5)), + ] + # 3 turns in 5 seconds = 36 turns/min + result = await rule.evaluate(case, dialog) + assert result.passed is True + assert "吞吐量" in result.reason + assert "turns/min" in result.reason + + +@pytest.mark.asyncio +async def test_response_time_throughput_exceeded(): + """Throughput below threshold should fail.""" + rule = get_rule("response_time", {"throughput_min": 100}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + base_time = datetime(2026, 1, 1, 12, 0, 0) + dialog = [ + _make_turn(1, 1000, sent_at=base_time, received_at=base_time + timedelta(seconds=10)), + _make_turn(2, 1000, sent_at=base_time + timedelta(seconds=20), received_at=base_time + timedelta(seconds=30)), + ] + # 2 turns in 30 seconds = 4 turns/min < 100 + result = await rule.evaluate(case, dialog) + assert result.passed is False + assert "未通过" in result.reason + + +@pytest.mark.asyncio +async def test_response_time_multiple_metrics(): + """Multiple metrics can be checked together.""" + rule = get_rule("response_time", { + "max_ms": 5000, + "avg_latency_max_ms": 4000, + }) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + dialog = [ + _make_turn(1, 3000), + _make_turn(2, 4500), + ] + result = await rule.evaluate(case, dialog) + assert result.passed is True + assert "最后一轮" in result.reason + assert "平均延迟" in result.reason + + +@pytest.mark.asyncio +async def test_response_time_details_field(): + """Result should include details with latency statistics.""" + rule = get_rule("response_time", {"max_ms": 5000}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + dialog = [ + _make_turn(1, 2000), + _make_turn(2, 3000), + _make_turn(3, 4000), + ] + result = await rule.evaluate(case, dialog) + assert result.details is not None + assert result.details["latencies"] == [2000, 3000, 4000] + assert result.details["avg_latency_ms"] == 3000 + assert result.details["min_latency_ms"] == 2000 + assert result.details["max_latency_ms"] == 4000 + + +@pytest.mark.asyncio +async def test_response_time_no_threshold(): + """No threshold configured should pass with basic info.""" + rule = get_rule("response_time", {}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + dialog = [_make_turn(1, 3000)] + result = await rule.evaluate(case, dialog) + assert result.passed is True + assert "3000ms" in result.reason + + +@pytest.mark.asyncio +async def test_response_time_empty_dialog(): + """Empty dialog should fail.""" + rule = get_rule("response_time", {"max_ms": 5000}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + result = await rule.evaluate(case, []) + assert result.passed is False + assert "无回复记录" in result.reason