Merge pull request 'feat(response_time): 支持平均延迟和吞吐量指标' (#29) from feat/response-time-extended into main

backend/agenteval/evaluation/rules/response_time.py

@@ -6,7 +6,13 @@ from agenteval.models import Case, Turn
 
 @register_rule
 class ResponseTimeRule(EvalRule):
-    """Check whether the reply latency is within the configured threshold."""
+    """Check whether the reply latency is within the configured threshold.
+
+    Supports multiple metrics:
+    - max_ms: threshold for single turn latency (backward compatible)
+    - avg_latency_max_ms: threshold for average latency across all turns
+    - throughput_min: minimum throughput in turns per minute
+    """
 
     name = "response_time"
 
@@ -14,23 +20,71 @@ class ResponseTimeRule(EvalRule):
         if not dialog:
             return RuleResult(passed=False, reason="无回复记录")
 
+        # Collect latencies from all turns
+        latencies = [t.latency_ms for t in dialog if t.latency_ms is not None]
+        if not latencies:
+            return RuleResult(passed=False, reason="无法获取响应时间")
+
+        # Backward compatible: check last turn against max_ms threshold
         threshold_ms = self.params.get("max_ms")
         if threshold_ms is None:
             threshold_ms = case.expectations.response_time_max_ms
-        if threshold_ms is None:
-            return RuleResult(passed=True, reason="未配置响应时间阈值")
 
-        last_turn = dialog[-1]
+        results = []
-        latency = last_turn.latency_ms
+        all_passed = True
-        if latency is None:
-            return RuleResult(passed=False, reason="无法获取响应时间")
 
-        if latency > threshold_ms:
+        # 1. Single turn latency check (backward compatible)
-            return RuleResult(
+        if threshold_ms is not None:
-                passed=False,
+            last_latency = latencies[-1]
-                score=max(0.0, 1.0 - (latency - threshold_ms) / threshold_ms),
+            passed = last_latency <= threshold_ms
-                reason=f"响应时间 {latency}ms 超过阈值 {threshold_ms}ms",
+            if not passed:
-            )
+                all_passed = False
+            results.append(f"最后一轮 {last_latency}ms {'≤' if passed else '>'} {threshold_ms}ms")
 
-        score = 1.0 if latency <= 0 else min(1.0, threshold_ms / latency)
+        # 2. Average latency check
-        return RuleResult(passed=True, score=score, reason=f"响应时间 {latency}ms 通过")
+        avg_latency_max_ms = self.params.get("avg_latency_max_ms")
+        if avg_latency_max_ms is not None:
+            avg_latency = sum(latencies) / len(latencies)
+            passed = avg_latency <= avg_latency_max_ms
+            if not passed:
+                all_passed = False
+            results.append(f"平均延迟 {avg_latency:.0f}ms {'≤' if passed else '>'} {avg_latency_max_ms}ms")
+
+        # 3. Throughput check (turns per minute)
+        throughput_min = self.params.get("throughput_min")
+        if throughput_min is not None and len(dialog) >= 2:
+            first_sent = dialog[0].sent_at
+            last_received = dialog[-1].received_at
+            if first_sent and last_received:
+                duration_minutes = (last_received - first_sent).total_seconds() / 60
+                if duration_minutes > 0:
+                    throughput = len(dialog) / duration_minutes
+                    passed = throughput >= throughput_min
+                    if not passed:
+                        all_passed = False
+                    results.append(f"吞吐量 {throughput:.1f} turns/min {'≥' if passed else '<'} {throughput_min}")
+
+        # If no metrics configured, just report the last turn latency
+        if not results:
+            last_latency = latencies[-1]
+            return RuleResult(passed=True, score=1.0, reason=f"响应时间 {last_latency}ms")
+
+        # Calculate score based on average latency
+        avg_latency = sum(latencies) / len(latencies)
+        if threshold_ms:
+            score = 1.0 if avg_latency <= 0 else min(1.0, threshold_ms / avg_latency)
+        else:
+            score = 1.0
+
+        verdict = "通过" if all_passed else "未通过"
+        return RuleResult(
+            passed=all_passed,
+            score=score,
+            reason=f"响应时间指标 {verdict}：{'; '.join(results)}",
+            details={
+                "latencies": latencies,
+                "avg_latency_ms": avg_latency,
+                "min_latency_ms": min(latencies),
+                "max_latency_ms": max(latencies),
+            },
+        )

tests/unit/test_response_time_extended.py

@@ -0,0 +1,164 @@
+"""Tests for extended response_time rule with multiple metrics."""
+
+from datetime import datetime, timedelta
+
+import pytest
+
+from agenteval.evaluation.rules.base import get_rule
+from agenteval.models import Case, Expectation, Turn
+
+
+def _make_turn(round_index: int, latency_ms: int, sent_at: datetime | None = None, received_at: datetime | None = None) -> Turn:
+    """Helper to create a Turn with latency."""
+    return Turn(
+        id=f"t-{round_index}",
+        run_id="r1",
+        case_id="c1",
+        round_index=round_index,
+        latency_ms=latency_ms,
+        sent_at=sent_at,
+        received_at=received_at,
+    )
+
+
+@pytest.mark.asyncio
+async def test_response_time_backward_compatible():
+    """Single turn with max_ms threshold should work as before."""
+    rule = get_rule("response_time", {"max_ms": 5000})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    dialog = [_make_turn(1, 3000)]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is True
+    assert "3000ms" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_response_time_exceeds_threshold():
+    """Single turn exceeding threshold should fail."""
+    rule = get_rule("response_time", {"max_ms": 2000})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    dialog = [_make_turn(1, 3000)]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is False
+    assert "未通过" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_response_time_avg_latency():
+    """Average latency check across multiple turns."""
+    rule = get_rule("response_time", {"avg_latency_max_ms": 3000})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    dialog = [
+        _make_turn(1, 2000),
+        _make_turn(2, 4000),
+    ]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is True  # avg = 3000, threshold = 3000
+    assert "平均延迟" in result.reason
+    assert result.details is not None
+    assert result.details["avg_latency_ms"] == 3000
+
+
+@pytest.mark.asyncio
+async def test_response_time_avg_latency_exceeded():
+    """Average latency exceeding threshold should fail."""
+    rule = get_rule("response_time", {"avg_latency_max_ms": 2000})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    dialog = [
+        _make_turn(1, 3000),
+        _make_turn(2, 4000),
+    ]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is False  # avg = 3500 > 2000
+    assert "未通过" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_response_time_throughput():
+    """Throughput check (turns per minute)."""
+    rule = get_rule("response_time", {"throughput_min": 10})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    base_time = datetime(2026, 1, 1, 12, 0, 0)
+    dialog = [
+        _make_turn(1, 1000, sent_at=base_time, received_at=base_time + timedelta(seconds=1)),
+        _make_turn(2, 1000, sent_at=base_time + timedelta(seconds=2), received_at=base_time + timedelta(seconds=3)),
+        _make_turn(3, 1000, sent_at=base_time + timedelta(seconds=4), received_at=base_time + timedelta(seconds=5)),
+    ]
+    # 3 turns in 5 seconds = 36 turns/min
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is True
+    assert "吞吐量" in result.reason
+    assert "turns/min" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_response_time_throughput_exceeded():
+    """Throughput below threshold should fail."""
+    rule = get_rule("response_time", {"throughput_min": 100})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    base_time = datetime(2026, 1, 1, 12, 0, 0)
+    dialog = [
+        _make_turn(1, 1000, sent_at=base_time, received_at=base_time + timedelta(seconds=10)),
+        _make_turn(2, 1000, sent_at=base_time + timedelta(seconds=20), received_at=base_time + timedelta(seconds=30)),
+    ]
+    # 2 turns in 30 seconds = 4 turns/min < 100
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is False
+    assert "未通过" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_response_time_multiple_metrics():
+    """Multiple metrics can be checked together."""
+    rule = get_rule("response_time", {
+        "max_ms": 5000,
+        "avg_latency_max_ms": 4000,
+    })
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    dialog = [
+        _make_turn(1, 3000),
+        _make_turn(2, 4500),
+    ]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is True
+    assert "最后一轮" in result.reason
+    assert "平均延迟" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_response_time_details_field():
+    """Result should include details with latency statistics."""
+    rule = get_rule("response_time", {"max_ms": 5000})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    dialog = [
+        _make_turn(1, 2000),
+        _make_turn(2, 3000),
+        _make_turn(3, 4000),
+    ]
+    result = await rule.evaluate(case, dialog)
+    assert result.details is not None
+    assert result.details["latencies"] == [2000, 3000, 4000]
+    assert result.details["avg_latency_ms"] == 3000
+    assert result.details["min_latency_ms"] == 2000
+    assert result.details["max_latency_ms"] == 4000
+
+
+@pytest.mark.asyncio
+async def test_response_time_no_threshold():
+    """No threshold configured should pass with basic info."""
+    rule = get_rule("response_time", {})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    dialog = [_make_turn(1, 3000)]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is True
+    assert "3000ms" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_response_time_empty_dialog():
+    """Empty dialog should fail."""
+    rule = get_rule("response_time", {"max_ms": 5000})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    result = await rule.evaluate(case, [])
+    assert result.passed is False
+    assert "无回复记录" in result.reason