c7f1dca49d
## 新规则（共 6 种，增加 3 种） ### semantic_similarity - 调用 OpenAI 兼容 embedding API（asyncio.gather 并发两路请求） - 余弦相似度与 reference 比对，min_score 可配置（默认 0.7） - API 异常时明确返回失败原因，不隐藏错误 ### json_schema - 验证回复是否为合法 JSON（支持 markdown 代码块剥离） - required_keys / forbidden_keys / key_types 三维校验 - dot-path 支持嵌套字段（"data.id"） - strict_json=false 模式非阻断校验 ### safety - 双层检测：关键词黑名单（零延迟）+ 可选 moderation API - API 不可用时自动降级黑名单，不中止评测 - 支持自定义 flagged_categories ## 规则组合逻辑（rule_logic + rule_pass_threshold） - models.py: EvalRuleConfig 增加 weight 字段；Case 增加 rule_logic / rule_pass_threshold - models.py: 新增 RuleLogic 枚举（all / any / weighted） - engine._save_rule_results: 按 rule_logic 决定 case 通过/失败 - ALL：全部通过才通过（原有行为，向下兼容） - ANY：至少一条通过即通过 - WEIGHTED：加权平均分 >= rule_pass_threshold ## 测试（43 → 76，新增 33） - test_s2_rules_and_logic.py：3 个新规则的 pass/fail/边界/API 降级 + 5 个组合逻辑集成测试 Co-Authored-By: Claude <noreply@anthropic.com>
95 lines
3.3 KiB
Python
95 lines
3.3 KiB
Python
"""Semantic similarity evaluation rule.
|
||
|
||
Uses an external embedding API to compute cosine similarity between the
|
||
agent reply and a reference answer. Requires an OpenAI-compatible
|
||
embeddings endpoint (POST /v1/embeddings or equivalent).
|
||
|
||
Configuration params:
|
||
api_url Embeddings API endpoint (required)
|
||
api_key Bearer token (optional)
|
||
model Embedding model name (default: text-embedding-3-small)
|
||
reference Reference text to compare against (required)
|
||
min_score Minimum cosine similarity to pass, 0-1 (default: 0.7)
|
||
"""
|
||
|
||
import asyncio
|
||
import math
|
||
|
||
import httpx
|
||
|
||
from agenteval.evaluation.rules.base import EvalRule, RuleResult, register_rule
|
||
from agenteval.models import Case, Turn
|
||
from agenteval.utils.llm import extract_reply_text
|
||
|
||
|
||
def _cosine(a: list[float], b: list[float]) -> float:
|
||
dot = sum(x * y for x, y in zip(a, b))
|
||
norm_a = math.sqrt(sum(x * x for x in a))
|
||
norm_b = math.sqrt(sum(x * x for x in b))
|
||
if norm_a == 0 or norm_b == 0:
|
||
return 0.0
|
||
return dot / (norm_a * norm_b)
|
||
|
||
|
||
async def _embed(
|
||
client: httpx.AsyncClient,
|
||
api_url: str,
|
||
api_key: str | None,
|
||
model: str,
|
||
text: str,
|
||
) -> list[float]:
|
||
headers = {"Content-Type": "application/json"}
|
||
if api_key:
|
||
headers["Authorization"] = f"Bearer {api_key}"
|
||
resp = await client.post(
|
||
api_url,
|
||
headers=headers,
|
||
json={"model": model, "input": text},
|
||
timeout=30,
|
||
)
|
||
resp.raise_for_status()
|
||
data = resp.json()
|
||
return data["data"][0]["embedding"]
|
||
|
||
|
||
@register_rule
|
||
class SemanticSimilarityRule(EvalRule):
|
||
"""Score reply by cosine similarity to a reference answer via embedding API."""
|
||
|
||
name = "semantic_similarity"
|
||
|
||
async def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult:
|
||
if not dialog:
|
||
return RuleResult(passed=False, reason="无回复记录")
|
||
|
||
reply_text = extract_reply_text(dialog[-1].reply)
|
||
if not reply_text:
|
||
return RuleResult(passed=False, reason="回复内容为空")
|
||
|
||
api_url: str | None = self.params.get("api_url")
|
||
api_key: str | None = self.params.get("api_key")
|
||
model: str = self.params.get("model", "text-embedding-3-small")
|
||
reference: str | None = self.params.get("reference")
|
||
min_score: float = float(self.params.get("min_score", 0.7))
|
||
|
||
if not api_url:
|
||
return RuleResult(passed=False, reason="semantic_similarity 未配置 api_url")
|
||
if not reference:
|
||
return RuleResult(passed=False, reason="semantic_similarity 未配置 reference")
|
||
|
||
try:
|
||
async with httpx.AsyncClient() as client:
|
||
reply_vec, ref_vec = await asyncio.gather(
|
||
_embed(client, api_url, api_key, model, reply_text),
|
||
_embed(client, api_url, api_key, model, reference),
|
||
)
|
||
similarity = _cosine(reply_vec, ref_vec)
|
||
passed = similarity >= min_score
|
||
return RuleResult(
|
||
passed=passed,
|
||
score=round(similarity, 4),
|
||
reason=f"语义相似度 {similarity:.3f}（阈值 {min_score}）",
|
||
)
|
||
except Exception as exc:
|
||
return RuleResult(passed=False, reason=f"embedding 调用失败: {exc}")
|