Merge pull request 'feat(llm_score): 支持多维度独立评分' (#27) from feat/llm-score-multi-dimension into main

backend/agenteval/evaluation/rules/base.py

@@ -18,6 +18,7 @@ class RuleResult:
     passed: bool
     score: Optional[float] = None
     reason: str = ""
+    details: Optional[dict[str, Any]] = None
 
 
 class EvalRule(ABC):

backend/agenteval/evaluation/rules/llm_score.py

@@ -1,5 +1,6 @@
 """LLM-based scoring evaluation rule."""
 
+import asyncio
 import json
 
 import httpx
@@ -22,9 +23,6 @@ class LlmScoreRule(EvalRule):
         last_turn = dialog[-1]
         reply_text = extract_reply_text(last_turn.reply)
 
-        # 用户问题取自当前轮发送的消息（sent_message），而非上一轮的智能体回复。
-        # 旧逻辑用 dialog[-2].reply 会把「上一轮 AI 回复」误当成「用户问题」，
-        # 导致多轮/动态用例里评分 LLM 收到牛头不对马嘴的问答对，普遍打 0 分。
         question_text = ""
         if last_turn.sent_message:
             body = last_turn.sent_message.get("msgBody", "")
@@ -36,6 +34,10 @@ class LlmScoreRule(EvalRule):
                 except Exception:
                     question_text = str(body)
 
+        dimensions = self.params.get("dimensions")
+        if dimensions:
+            return await self._evaluate_dimensions(question_text, reply_text, dimensions)
+
         criteria = self.params.get("criteria", "")
         min_score = float(self.params.get("min_score", 7))
         if self.model_config and self.gateway:
@@ -59,6 +61,55 @@ class LlmScoreRule(EvalRule):
             reason=f"LLM 评分 {score}/10，{verdict} (阈值 {min_score}){detail}",
         )
 
+    async def _evaluate_dimensions(
+        self, question: str, reply: str, dimensions: list[dict]
+    ) -> RuleResult:
+        tasks = [self._evaluate_one_dimension(question, reply, dim) for dim in dimensions]
+        results = await asyncio.gather(*tasks)
+
+        dimension_scores = {}
+        dimension_reasons = []
+        all_passed = True
+
+        for dim, (score, reason) in zip(dimensions, results):
+            dim_name = dim.get("name", "unknown")
+            min_score = float(dim.get("min_score", 7))
+            if score is None:
+                all_passed = False
+                dimension_scores[dim_name] = None
+                dimension_reasons.append(f"{dim_name}: 评分失败 ({reason})")
+            else:
+                passed = score >= min_score
+                if not passed:
+                    all_passed = False
+                dimension_scores[dim_name] = score
+                dimension_reasons.append(f"{dim_name}: {score}/10")
+
+        valid_scores = [s for s in dimension_scores.values() if s is not None]
+        avg_score = sum(valid_scores) / len(valid_scores) if valid_scores else 0
+
+        verdict = "通过" if all_passed else "未通过"
+        reasons_str = "，".join(dimension_reasons)
+        return RuleResult(
+            passed=all_passed,
+            score=avg_score / 10.0,
+            reason=f"多维度 LLM 评分 {avg_score:.1f}/10，{verdict}；{reasons_str}",
+            details={"dimensions": dimension_scores},
+        )
+
+    async def _evaluate_one_dimension(
+        self, question: str, reply: str, dimension: dict
+    ) -> tuple[float | None, str]:
+        criteria = dimension.get("criteria", "")
+        if self.model_config and self.gateway:
+            return await self._call_gateway(question, reply, criteria)
+        api_url = self.params.get("api_url")
+        api_key = self.params.get("api_key")
+        model = self.params.get("model", "gpt-4o-mini")
+        if not api_url:
+            return None, "未绑定评估模型"
+        return await self._call_llm(api_url, api_key, model, question, reply, criteria)
+
     async def _call_gateway(self, question: str, reply: str, criteria: str) -> tuple[float | None, str]:
         system_prompt, user_prompt = self._prompts(question, reply, criteria)
         try:

tests/unit/test_llm_score_multi_dimension.py

@@ -0,0 +1,66 @@
+"""Tests for multi-dimensional LLM scoring."""
+
+import pytest
+
+from agenteval.evaluation.rules.base import get_rule
+from agenteval.models import Case, Turn
+
+
+@pytest.mark.asyncio
+async def test_llm_score_single_dimension_backward_compatible():
+    """Single-dimension mode (criteria param) should work as before."""
+    rule = get_rule(
+        "llm_score",
+        {"criteria": "准确性", "min_score": 7, "api_url": "http://mock"},
+    )
+    assert rule.params["criteria"] == "准确性"
+    assert "dimensions" not in rule.params
+
+
+@pytest.mark.asyncio
+async def test_llm_score_multi_dimension_config():
+    """Multi-dimension mode should accept dimensions parameter."""
+    rule = get_rule(
+        "llm_score",
+        {
+            "dimensions": [
+                {"name": "accuracy", "criteria": "回答是否准确", "min_score": 7},
+                {"name": "relevance", "criteria": "回答是否相关", "min_score": 7},
+                {"name": "completeness", "criteria": "回答是否完整", "min_score": 7},
+            ]
+        },
+    )
+    assert len(rule.params["dimensions"]) == 3
+    assert rule.params["dimensions"][0]["name"] == "accuracy"
+
+
+@pytest.mark.asyncio
+async def test_llm_score_multi_dimension_evaluate_empty_dialog():
+    """Multi-dimension mode should handle empty dialog."""
+    rule = get_rule(
+        "llm_score",
+        {
+            "dimensions": [
+                {"name": "accuracy", "criteria": "准确性", "min_score": 7},
+            ]
+        },
+    )
+    case = Case(id="c1", messages=["hello"])
+    result = await rule.evaluate(case, [])
+    assert result.passed is False
+    assert "无回复记录" in result.reason
+
+
+@pytest.mark.asyncio
+async def test_rule_result_has_details_field():
+    """RuleResult should support details field for multi-dimensional scores."""
+    from agenteval.evaluation.rules.base import RuleResult
+
+    result = RuleResult(
+        passed=True,
+        score=0.8,
+        reason="test",
+        details={"dimensions": {"accuracy": 8, "relevance": 9}},
+    )
+    assert result.details is not None
+    assert result.details["dimensions"]["accuracy"] == 8