Merge pull request 'fix(rules): v1.3.1 Phase 1 修复规则层三处缺陷' (#33) from fix/v1.3.1-phase1-rule-bugfixes into main

Reviewed-on: #33

backend/agenteval/evaluation/rules/llm_score.py

@@ -61,11 +61,18 @@ class LlmScoreRule(EvalRule):
             reason=f"LLM 评分 {score}/10，{verdict} (阈值 {min_score}){detail}",
         )
 
+    _MAX_CONCURRENT_DIMENSIONS = 5
+
     async def _evaluate_dimensions(
         self, question: str, reply: str, dimensions: list[dict]
     ) -> RuleResult:
-        tasks = [self._evaluate_one_dimension(question, reply, dim) for dim in dimensions]
-        results = await asyncio.gather(*tasks)
+        semaphore = asyncio.Semaphore(self._MAX_CONCURRENT_DIMENSIONS)
+
+        async def bounded(dim: dict) -> tuple[float | None, str]:
+            async with semaphore:
+                return await self._evaluate_one_dimension(question, reply, dim)
+
+        results = await asyncio.gather(*(bounded(dim) for dim in dimensions))
 
         dimension_scores = {}
         dimension_reasons = []

backend/agenteval/evaluation/rules/response_time.py

@@ -69,9 +69,16 @@ class ResponseTimeRule(EvalRule):
             last_latency = latencies[-1]
             return RuleResult(passed=True, score=1.0, reason=f"响应时间 {last_latency}ms")
 
-        # Calculate score based on average latency
         avg_latency = sum(latencies) / len(latencies)
-        if threshold_ms:
+        has_extended_metrics = avg_latency_max_ms is not None or throughput_min is not None
+        if threshold_ms is not None and not has_extended_metrics:
+            # 仅 max_ms：保持 v0.3 原语义——基于最后一轮评分，超限有惩罚
+            last_latency = latencies[-1]
+            if last_latency > threshold_ms:
+                score = max(0.0, 1.0 - (last_latency - threshold_ms) / threshold_ms)
+            else:
+                score = 1.0 if last_latency <= 0 else min(1.0, threshold_ms / last_latency)
+        elif threshold_ms is not None:
             score = 1.0 if avg_latency <= 0 else min(1.0, threshold_ms / avg_latency)
         else:
             score = 1.0

backend/agenteval/evaluation/rules/safety.py

@@ -69,7 +69,7 @@ class SafetyRule(EvalRule):
             issues.append(f"包含违禁词: {hit_words}")
 
         # Layer 2: moderation API (optional, degrades gracefully)
-        if use_api and (api_url or self.model_config) and not hit_words:
+        if use_api and (api_url or self.model_config):
             try:
                 if self.model_config and self.gateway:
                     result = await self.gateway.moderate(self.model_config, reply_text)

tests/unit/test_llm_score_multi_dimension.py

@@ -64,3 +64,34 @@ async def test_rule_result_has_details_field():
     )
     assert result.details is not None
     assert result.details["dimensions"]["accuracy"] == 8
+
+
+@pytest.mark.asyncio
+async def test_llm_score_multi_dimension_concurrency_bounded(monkeypatch):
+    """多维度评分的并行 LLM 调用数不得超过 _MAX_CONCURRENT_DIMENSIONS。"""
+    import asyncio
+
+    from agenteval.evaluation.rules.llm_score import LlmScoreRule
+
+    in_flight = 0
+    peak = 0
+
+    async def fake_dimension(self, question, reply, dimension):
+        nonlocal in_flight, peak
+        in_flight += 1
+        peak = max(peak, in_flight)
+        await asyncio.sleep(0.01)
+        in_flight -= 1
+        return 8.0, "ok"
+
+    monkeypatch.setattr(LlmScoreRule, "_evaluate_one_dimension", fake_dimension)
+
+    rule = get_rule(
+        "llm_score",
+        {"dimensions": [{"name": f"dim{i}", "criteria": "c", "min_score": 7} for i in range(8)]},
+    )
+    case = Case(id="c1", messages=["hello"])
+    dialog = [Turn(id="t-1", run_id="r1", case_id="c1", round_index=1, reply={"msgBody": "回答"})]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is True
+    assert peak <= LlmScoreRule._MAX_CONCURRENT_DIMENSIONS

tests/unit/test_response_time_extended.py

@@ -162,3 +162,37 @@ async def test_response_time_empty_dialog():
     result = await rule.evaluate(case, [])
     assert result.passed is False
     assert "无回复记录" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_response_time_max_ms_only_exceed_penalty():
+    """仅 max_ms 时保持 v0.3 语义：超限有线性惩罚（score = 1 - 超出/阈值）。"""
+    rule = get_rule("response_time", {"max_ms": 2000})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    dialog = [_make_turn(1, 3000)]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is False
+    assert result.score == pytest.approx(0.5)  # 1 - (3000-2000)/2000
+
+
+@pytest.mark.asyncio
+async def test_response_time_max_ms_only_last_turn_decides():
+    """仅 max_ms 时保持 v0.3 语义：最后一轮决定评分，即使平均更快。"""
+    rule = get_rule("response_time", {"max_ms": 6000})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    # 最后一轮 5000ms 未超限 → score 1.0（平均 3000ms 不影响）
+    dialog = [_make_turn(1, 1000), _make_turn(2, 5000)]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is True
+    assert result.score == pytest.approx(1.0)
+
+
+@pytest.mark.asyncio
+async def test_response_time_mixed_metrics_uses_average_score():
+    """配置了扩展指标时 score 基于平均延迟。"""
+    rule = get_rule("response_time", {"max_ms": 6000, "avg_latency_max_ms": 4000})
+    case = Case(id="c1", messages=["hello"], expectations=Expectation())
+    dialog = [_make_turn(1, 2000), _make_turn(2, 4000)]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is True
+    assert result.score == pytest.approx(1.0)  # avg=3000 <= 6000 → min(1, 6000/3000)=1

tests/unit/test_safety_extended.py

@@ -140,3 +140,27 @@ async def test_safety_empty_reply():
     result = await rule.evaluate(case, dialog)
     assert result.passed is True
     assert "空回复" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_safety_moderation_runs_even_when_blacklist_hits(monkeypatch):
+    """黑名单命中时 moderation API 仍应执行，两层发现都要上报。"""
+    from agenteval.evaluation.rules.safety import SafetyRule
+
+    async def fake_moderation(self, api_url, api_key, text, flagged_categories):
+        return True, ["violence"]
+
+    monkeypatch.setattr(SafetyRule, "_call_moderation", fake_moderation)
+
+    rule = get_rule(
+        "safety",
+        {"blacklist": ["违禁词"], "use_moderation_api": True, "api_url": "http://mock/moderations"},
+    )
+    case = Case(id="c1", messages=["hello"])
+    dialog = [_make_turn_with_reply("这个回复包含违禁词")]
+    result = await rule.evaluate(case, dialog)
+    assert result.passed is False
+    assert result.details is not None
+    issues = result.details["issues"]
+    assert any("违禁词" in i for i in issues)
+    assert any("moderation API 标记" in i for i in issues)