From c1a3cdbaa9817a86a50f891d14ea35f2b0aa0ff2 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Tue, 25 Aug 2026 18:13:11 +0800 Subject: [PATCH] =?UTF-8?q?fix(rules):=20=E4=BF=AE=E5=A4=8D=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=AE=A1=E6=9F=A5=E5=8F=91=E7=8E=B0=E7=9A=84=E4=B8=89?= =?UTF-8?q?=E5=A4=84=E8=A7=84=E5=88=99=E5=B1=82=E7=BC=BA=E9=99=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - response_time: 仅配置 max_ms 时恢复 v0.3 评分语义（最后一轮评分 + 超限线性惩罚），扩展指标共存时才用均值评分 - safety: 移除 moderation API 的黑名单命中跳过守卫，两层安全检查独立执行、发现均上报 - llm_score: 多维度评分添加 Semaphore 并发上限（5），防止维度数多时无限扇出模型请求 --- .../agenteval/evaluation/rules/llm_score.py | 11 ++++-- .../evaluation/rules/response_time.py | 11 ++++-- backend/agenteval/evaluation/rules/safety.py | 2 +- tests/unit/test_llm_score_multi_dimension.py | 31 +++++++++++++++++ tests/unit/test_response_time_extended.py | 34 +++++++++++++++++++ tests/unit/test_safety_extended.py | 24 +++++++++++++ 6 files changed, 108 insertions(+), 5 deletions(-) diff --git a/backend/agenteval/evaluation/rules/llm_score.py b/backend/agenteval/evaluation/rules/llm_score.py index 72a3857..9ec0ae7 100644 --- a/backend/agenteval/evaluation/rules/llm_score.py +++ b/backend/agenteval/evaluation/rules/llm_score.py @@ -61,11 +61,18 @@ class LlmScoreRule(EvalRule): reason=f"LLM 评分 {score}/10，{verdict} (阈值 {min_score}){detail}", ) + _MAX_CONCURRENT_DIMENSIONS = 5 + async def _evaluate_dimensions( self, question: str, reply: str, dimensions: list[dict] ) -> RuleResult: - tasks = [self._evaluate_one_dimension(question, reply, dim) for dim in dimensions] - results = await asyncio.gather(*tasks) + semaphore = asyncio.Semaphore(self._MAX_CONCURRENT_DIMENSIONS) + + async def bounded(dim: dict) -> tuple[float | None, str]: + async with semaphore: + return await self._evaluate_one_dimension(question, reply, dim) + + results = await asyncio.gather(*(bounded(dim) for dim in dimensions)) dimension_scores = {} dimension_reasons = [] diff --git a/backend/agenteval/evaluation/rules/response_time.py b/backend/agenteval/evaluation/rules/response_time.py index 652a8f2..a6e5dd2 100644 --- a/backend/agenteval/evaluation/rules/response_time.py +++ b/backend/agenteval/evaluation/rules/response_time.py @@ -69,9 +69,16 @@ class ResponseTimeRule(EvalRule): last_latency = latencies[-1] return RuleResult(passed=True, score=1.0, reason=f"响应时间 {last_latency}ms") - # Calculate score based on average latency avg_latency = sum(latencies) / len(latencies) - if threshold_ms: + has_extended_metrics = avg_latency_max_ms is not None or throughput_min is not None + if threshold_ms is not None and not has_extended_metrics: + # 仅 max_ms：保持 v0.3 原语义——基于最后一轮评分，超限有惩罚 + last_latency = latencies[-1] + if last_latency > threshold_ms: + score = max(0.0, 1.0 - (last_latency - threshold_ms) / threshold_ms) + else: + score = 1.0 if last_latency <= 0 else min(1.0, threshold_ms / last_latency) + elif threshold_ms is not None: score = 1.0 if avg_latency <= 0 else min(1.0, threshold_ms / avg_latency) else: score = 1.0 diff --git a/backend/agenteval/evaluation/rules/safety.py b/backend/agenteval/evaluation/rules/safety.py index 47d44e9..0f14a8b 100644 --- a/backend/agenteval/evaluation/rules/safety.py +++ b/backend/agenteval/evaluation/rules/safety.py @@ -69,7 +69,7 @@ class SafetyRule(EvalRule): issues.append(f"包含违禁词: {hit_words}") # Layer 2: moderation API (optional, degrades gracefully) - if use_api and (api_url or self.model_config) and not hit_words: + if use_api and (api_url or self.model_config): try: if self.model_config and self.gateway: result = await self.gateway.moderate(self.model_config, reply_text) diff --git a/tests/unit/test_llm_score_multi_dimension.py b/tests/unit/test_llm_score_multi_dimension.py index 8888793..291e62e 100644 --- a/tests/unit/test_llm_score_multi_dimension.py +++ b/tests/unit/test_llm_score_multi_dimension.py @@ -64,3 +64,34 @@ async def test_rule_result_has_details_field(): ) assert result.details is not None assert result.details["dimensions"]["accuracy"] == 8 + + +@pytest.mark.asyncio +async def test_llm_score_multi_dimension_concurrency_bounded(monkeypatch): + """多维度评分的并行 LLM 调用数不得超过 _MAX_CONCURRENT_DIMENSIONS。""" + import asyncio + + from agenteval.evaluation.rules.llm_score import LlmScoreRule + + in_flight = 0 + peak = 0 + + async def fake_dimension(self, question, reply, dimension): + nonlocal in_flight, peak + in_flight += 1 + peak = max(peak, in_flight) + await asyncio.sleep(0.01) + in_flight -= 1 + return 8.0, "ok" + + monkeypatch.setattr(LlmScoreRule, "_evaluate_one_dimension", fake_dimension) + + rule = get_rule( + "llm_score", + {"dimensions": [{"name": f"dim{i}", "criteria": "c", "min_score": 7} for i in range(8)]}, + ) + case = Case(id="c1", messages=["hello"]) + dialog = [Turn(id="t-1", run_id="r1", case_id="c1", round_index=1, reply={"msgBody": "回答"})] + result = await rule.evaluate(case, dialog) + assert result.passed is True + assert peak <= LlmScoreRule._MAX_CONCURRENT_DIMENSIONS diff --git a/tests/unit/test_response_time_extended.py b/tests/unit/test_response_time_extended.py index a20748c..6f4e571 100644 --- a/tests/unit/test_response_time_extended.py +++ b/tests/unit/test_response_time_extended.py @@ -162,3 +162,37 @@ async def test_response_time_empty_dialog(): result = await rule.evaluate(case, []) assert result.passed is False assert "无回复记录" in result.reason + + +@pytest.mark.asyncio +async def test_response_time_max_ms_only_exceed_penalty(): + """仅 max_ms 时保持 v0.3 语义：超限有线性惩罚（score = 1 - 超出/阈值）。""" + rule = get_rule("response_time", {"max_ms": 2000}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + dialog = [_make_turn(1, 3000)] + result = await rule.evaluate(case, dialog) + assert result.passed is False + assert result.score == pytest.approx(0.5) # 1 - (3000-2000)/2000 + + +@pytest.mark.asyncio +async def test_response_time_max_ms_only_last_turn_decides(): + """仅 max_ms 时保持 v0.3 语义：最后一轮决定评分，即使平均更快。""" + rule = get_rule("response_time", {"max_ms": 6000}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + # 最后一轮 5000ms 未超限 → score 1.0（平均 3000ms 不影响） + dialog = [_make_turn(1, 1000), _make_turn(2, 5000)] + result = await rule.evaluate(case, dialog) + assert result.passed is True + assert result.score == pytest.approx(1.0) + + +@pytest.mark.asyncio +async def test_response_time_mixed_metrics_uses_average_score(): + """配置了扩展指标时 score 基于平均延迟。""" + rule = get_rule("response_time", {"max_ms": 6000, "avg_latency_max_ms": 4000}) + case = Case(id="c1", messages=["hello"], expectations=Expectation()) + dialog = [_make_turn(1, 2000), _make_turn(2, 4000)] + result = await rule.evaluate(case, dialog) + assert result.passed is True + assert result.score == pytest.approx(1.0) # avg=3000 <= 6000 → min(1, 6000/3000)=1 diff --git a/tests/unit/test_safety_extended.py b/tests/unit/test_safety_extended.py index 13bdae1..23b08a6 100644 --- a/tests/unit/test_safety_extended.py +++ b/tests/unit/test_safety_extended.py @@ -140,3 +140,27 @@ async def test_safety_empty_reply(): result = await rule.evaluate(case, dialog) assert result.passed is True assert "空回复" in result.reason + + +@pytest.mark.asyncio +async def test_safety_moderation_runs_even_when_blacklist_hits(monkeypatch): + """黑名单命中时 moderation API 仍应执行，两层发现都要上报。""" + from agenteval.evaluation.rules.safety import SafetyRule + + async def fake_moderation(self, api_url, api_key, text, flagged_categories): + return True, ["violence"] + + monkeypatch.setattr(SafetyRule, "_call_moderation", fake_moderation) + + rule = get_rule( + "safety", + {"blacklist": ["违禁词"], "use_moderation_api": True, "api_url": "http://mock/moderations"}, + ) + case = Case(id="c1", messages=["hello"]) + dialog = [_make_turn_with_reply("这个回复包含违禁词")] + result = await rule.evaluate(case, dialog) + assert result.passed is False + assert result.details is not None + issues = result.details["issues"] + assert any("违禁词" in i for i in issues) + assert any("moderation API 标记" in i for i in issues) -- 2.45.2