From a29f78ff4b6317e8d58d5ebd829abf56a657a44e Mon Sep 17 00:00:00 2001 From: sinohqb Date: Tue, 25 Aug 2026 14:17:49 +0800 Subject: [PATCH] =?UTF-8?q?feat(llm=5Fscore):=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=A4=9A=E7=BB=B4=E5=BA=A6=E7=8B=AC=E7=AB=8B=E8=AF=84=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 扩展 llm_score 规则，支持通过 dimensions 参数配置多个评分维度， 每个维度独立评分 0-10 分。 - RuleResult 新增 details 字段存储结构化多维度分数 - 向后兼容：原有 criteria 单维度模式继续有效 - 多维度模式下各维度并行调用 LLM，返回平均分和明细 - 新增 4 项单元测试 Closes #19 --- backend/agenteval/evaluation/rules/base.py | 1 + .../agenteval/evaluation/rules/llm_score.py | 57 +++++++++++++++- tests/unit/test_llm_score_multi_dimension.py | 66 +++++++++++++++++++ 3 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_llm_score_multi_dimension.py diff --git a/backend/agenteval/evaluation/rules/base.py b/backend/agenteval/evaluation/rules/base.py index d032c4d..cb8059c 100644 --- a/backend/agenteval/evaluation/rules/base.py +++ b/backend/agenteval/evaluation/rules/base.py @@ -18,6 +18,7 @@ class RuleResult: passed: bool score: Optional[float] = None reason: str = "" + details: Optional[dict[str, Any]] = None class EvalRule(ABC): diff --git a/backend/agenteval/evaluation/rules/llm_score.py b/backend/agenteval/evaluation/rules/llm_score.py index 1ae5749..72a3857 100644 --- a/backend/agenteval/evaluation/rules/llm_score.py +++ b/backend/agenteval/evaluation/rules/llm_score.py @@ -1,5 +1,6 @@ """LLM-based scoring evaluation rule.""" +import asyncio import json import httpx @@ -22,9 +23,6 @@ class LlmScoreRule(EvalRule): last_turn = dialog[-1] reply_text = extract_reply_text(last_turn.reply) - # 用户问题取自当前轮发送的消息（sent_message），而非上一轮的智能体回复。 - # 旧逻辑用 dialog[-2].reply 会把「上一轮 AI 回复」误当成「用户问题」， - # 导致多轮/动态用例里评分 LLM 收到牛头不对马嘴的问答对，普遍打 0 分。 question_text = "" if last_turn.sent_message: body = last_turn.sent_message.get("msgBody", "") @@ -36,6 +34,10 @@ class LlmScoreRule(EvalRule): except Exception: question_text = str(body) + dimensions = self.params.get("dimensions") + if dimensions: + return await self._evaluate_dimensions(question_text, reply_text, dimensions) + criteria = self.params.get("criteria", "") min_score = float(self.params.get("min_score", 7)) if self.model_config and self.gateway: @@ -59,6 +61,55 @@ class LlmScoreRule(EvalRule): reason=f"LLM 评分 {score}/10，{verdict} (阈值 {min_score}){detail}", ) + async def _evaluate_dimensions( + self, question: str, reply: str, dimensions: list[dict] + ) -> RuleResult: + tasks = [self._evaluate_one_dimension(question, reply, dim) for dim in dimensions] + results = await asyncio.gather(*tasks) + + dimension_scores = {} + dimension_reasons = [] + all_passed = True + + for dim, (score, reason) in zip(dimensions, results): + dim_name = dim.get("name", "unknown") + min_score = float(dim.get("min_score", 7)) + if score is None: + all_passed = False + dimension_scores[dim_name] = None + dimension_reasons.append(f"{dim_name}: 评分失败 ({reason})") + else: + passed = score >= min_score + if not passed: + all_passed = False + dimension_scores[dim_name] = score + dimension_reasons.append(f"{dim_name}: {score}/10") + + valid_scores = [s for s in dimension_scores.values() if s is not None] + avg_score = sum(valid_scores) / len(valid_scores) if valid_scores else 0 + + verdict = "通过" if all_passed else "未通过" + reasons_str = "，".join(dimension_reasons) + return RuleResult( + passed=all_passed, + score=avg_score / 10.0, + reason=f"多维度 LLM 评分 {avg_score:.1f}/10，{verdict}；{reasons_str}", + details={"dimensions": dimension_scores}, + ) + + async def _evaluate_one_dimension( + self, question: str, reply: str, dimension: dict + ) -> tuple[float | None, str]: + criteria = dimension.get("criteria", "") + if self.model_config and self.gateway: + return await self._call_gateway(question, reply, criteria) + api_url = self.params.get("api_url") + api_key = self.params.get("api_key") + model = self.params.get("model", "gpt-4o-mini") + if not api_url: + return None, "未绑定评估模型" + return await self._call_llm(api_url, api_key, model, question, reply, criteria) + async def _call_gateway(self, question: str, reply: str, criteria: str) -> tuple[float | None, str]: system_prompt, user_prompt = self._prompts(question, reply, criteria) try: diff --git a/tests/unit/test_llm_score_multi_dimension.py b/tests/unit/test_llm_score_multi_dimension.py new file mode 100644 index 0000000..8888793 --- /dev/null +++ b/tests/unit/test_llm_score_multi_dimension.py @@ -0,0 +1,66 @@ +"""Tests for multi-dimensional LLM scoring.""" + +import pytest + +from agenteval.evaluation.rules.base import get_rule +from agenteval.models import Case, Turn + + +@pytest.mark.asyncio +async def test_llm_score_single_dimension_backward_compatible(): + """Single-dimension mode (criteria param) should work as before.""" + rule = get_rule( + "llm_score", + {"criteria": "准确性", "min_score": 7, "api_url": "http://mock"}, + ) + assert rule.params["criteria"] == "准确性" + assert "dimensions" not in rule.params + + +@pytest.mark.asyncio +async def test_llm_score_multi_dimension_config(): + """Multi-dimension mode should accept dimensions parameter.""" + rule = get_rule( + "llm_score", + { + "dimensions": [ + {"name": "accuracy", "criteria": "回答是否准确", "min_score": 7}, + {"name": "relevance", "criteria": "回答是否相关", "min_score": 7}, + {"name": "completeness", "criteria": "回答是否完整", "min_score": 7}, + ] + }, + ) + assert len(rule.params["dimensions"]) == 3 + assert rule.params["dimensions"][0]["name"] == "accuracy" + + +@pytest.mark.asyncio +async def test_llm_score_multi_dimension_evaluate_empty_dialog(): + """Multi-dimension mode should handle empty dialog.""" + rule = get_rule( + "llm_score", + { + "dimensions": [ + {"name": "accuracy", "criteria": "准确性", "min_score": 7}, + ] + }, + ) + case = Case(id="c1", messages=["hello"]) + result = await rule.evaluate(case, []) + assert result.passed is False + assert "无回复记录" in result.reason + + +@pytest.mark.asyncio +async def test_rule_result_has_details_field(): + """RuleResult should support details field for multi-dimensional scores.""" + from agenteval.evaluation.rules.base import RuleResult + + result = RuleResult( + passed=True, + score=0.8, + reason="test", + details={"dimensions": {"accuracy": 8, "relevance": 9}}, + ) + assert result.details is not None + assert result.details["dimensions"]["accuracy"] == 8