c7f1dca49d
## 新规则（共 6 种，增加 3 种） ### semantic_similarity - 调用 OpenAI 兼容 embedding API（asyncio.gather 并发两路请求） - 余弦相似度与 reference 比对，min_score 可配置（默认 0.7） - API 异常时明确返回失败原因，不隐藏错误 ### json_schema - 验证回复是否为合法 JSON（支持 markdown 代码块剥离） - required_keys / forbidden_keys / key_types 三维校验 - dot-path 支持嵌套字段（"data.id"） - strict_json=false 模式非阻断校验 ### safety - 双层检测：关键词黑名单（零延迟）+ 可选 moderation API - API 不可用时自动降级黑名单，不中止评测 - 支持自定义 flagged_categories ## 规则组合逻辑（rule_logic + rule_pass_threshold） - models.py: EvalRuleConfig 增加 weight 字段；Case 增加 rule_logic / rule_pass_threshold - models.py: 新增 RuleLogic 枚举（all / any / weighted） - engine._save_rule_results: 按 rule_logic 决定 case 通过/失败 - ALL：全部通过才通过（原有行为，向下兼容） - ANY：至少一条通过即通过 - WEIGHTED：加权平均分 >= rule_pass_threshold ## 测试（43 → 76，新增 33） - test_s2_rules_and_logic.py：3 个新规则的 pass/fail/边界/API 降级 + 5 个组合逻辑集成测试 Co-Authored-By: Claude <noreply@anthropic.com>
378 lines
14 KiB
Python
378 lines
14 KiB
Python
"""Tests for S2 new rules: json_schema, safety, semantic_similarity,
|
|
and rule combination logic (all/any/weighted)."""
|
|
|
|
import asyncio
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from agenteval.evaluation.rules.json_schema import JsonSchemaRule, _get_path
|
|
from agenteval.evaluation.rules.safety import SafetyRule
|
|
from agenteval.evaluation.rules.semantic import SemanticSimilarityRule, _cosine
|
|
from agenteval.models import Case, CaseType, EvalRuleConfig, Expectation, RuleLogic, Turn
|
|
|
|
|
|
# ── helpers ──────────────────────────────────────────────────────────────
|
|
|
|
def _turn(reply_text: str, latency_ms: int = 100) -> Turn:
|
|
return Turn(
|
|
id="t1", run_id="r1", case_id="c1", round_index=1,
|
|
reply={"msgBody": {"content": reply_text}},
|
|
latency_ms=latency_ms,
|
|
)
|
|
|
|
|
|
def _case(
|
|
*,
|
|
rules: list[dict] | None = None,
|
|
rule_logic: RuleLogic = RuleLogic.ALL,
|
|
rule_pass_threshold: float = 0.6,
|
|
) -> Case:
|
|
eval_rules = [EvalRuleConfig(**r) for r in (rules or [])]
|
|
return Case(
|
|
id="c1", type=CaseType.SINGLE, messages=["hi"],
|
|
eval_rules=eval_rules,
|
|
rule_logic=rule_logic,
|
|
rule_pass_threshold=rule_pass_threshold,
|
|
)
|
|
|
|
|
|
# ── _get_path ─────────────────────────────────────────────────────────────
|
|
|
|
def test_json_schema_get_path_nested():
|
|
assert _get_path({"a": {"b": 1}}, "a.b") == (True, 1)
|
|
|
|
|
|
def test_json_schema_get_path_missing():
|
|
assert _get_path({"a": 1}, "a.b") == (False, None)
|
|
|
|
|
|
# ── JsonSchemaRule ────────────────────────────────────────────────────────
|
|
|
|
async def test_json_schema_valid_json_no_constraints():
|
|
rule = JsonSchemaRule({})
|
|
result = await rule.evaluate(_case(), [_turn('{"key": "value"}')])
|
|
assert result.passed is True
|
|
|
|
|
|
async def test_json_schema_invalid_json_strict():
|
|
rule = JsonSchemaRule({"strict_json": True})
|
|
result = await rule.evaluate(_case(), [_turn("not json at all")])
|
|
assert result.passed is False
|
|
assert "合法 JSON" in result.reason
|
|
|
|
|
|
async def test_json_schema_invalid_json_nonstrict():
|
|
rule = JsonSchemaRule({"strict_json": False})
|
|
result = await rule.evaluate(_case(), [_turn("not json")])
|
|
assert result.passed is True
|
|
|
|
|
|
async def test_json_schema_required_key_present():
|
|
rule = JsonSchemaRule({"required_keys": ["status", "data.id"]})
|
|
result = await rule.evaluate(_case(), [_turn('{"status": "ok", "data": {"id": 42}}')])
|
|
assert result.passed is True
|
|
|
|
|
|
async def test_json_schema_required_key_missing():
|
|
rule = JsonSchemaRule({"required_keys": ["missing_key"]})
|
|
result = await rule.evaluate(_case(), [_turn('{"status": "ok"}')])
|
|
assert result.passed is False
|
|
assert "missing_key" in result.reason
|
|
|
|
|
|
async def test_json_schema_forbidden_key_present():
|
|
rule = JsonSchemaRule({"forbidden_keys": ["error"]})
|
|
result = await rule.evaluate(_case(), [_turn('{"status": "ok", "error": "oops"}')])
|
|
assert result.passed is False
|
|
assert "error" in result.reason
|
|
|
|
|
|
async def test_json_schema_type_check_pass():
|
|
rule = JsonSchemaRule({"key_types": {"count": "int", "name": "str"}})
|
|
result = await rule.evaluate(_case(), [_turn('{"count": 5, "name": "hello"}')])
|
|
assert result.passed is True
|
|
|
|
|
|
async def test_json_schema_type_check_fail():
|
|
rule = JsonSchemaRule({"key_types": {"count": "int"}})
|
|
result = await rule.evaluate(_case(), [_turn('{"count": "five"}')])
|
|
assert result.passed is False
|
|
assert "count" in result.reason
|
|
|
|
|
|
async def test_json_schema_strips_markdown_fence():
|
|
rule = JsonSchemaRule({"required_keys": ["id"]})
|
|
reply = '```json\n{"id": 123}\n```'
|
|
result = await rule.evaluate(_case(), [_turn(reply)])
|
|
assert result.passed is True
|
|
|
|
|
|
async def test_json_schema_empty_dialog():
|
|
rule = JsonSchemaRule({})
|
|
result = await rule.evaluate(_case(), [])
|
|
assert result.passed is False
|
|
|
|
|
|
# ── SafetyRule ────────────────────────────────────────────────────────────
|
|
|
|
async def test_safety_clean_reply():
|
|
rule = SafetyRule({"blacklist": ["kill", "harm"]})
|
|
result = await rule.evaluate(_case(), [_turn("This is a helpful response.")])
|
|
assert result.passed is True
|
|
|
|
|
|
async def test_safety_blacklist_hit():
|
|
rule = SafetyRule({"blacklist": ["kill"]})
|
|
result = await rule.evaluate(_case(), [_turn("I will kill you!")])
|
|
assert result.passed is False
|
|
assert "kill" in result.reason
|
|
|
|
|
|
async def test_safety_blacklist_case_insensitive():
|
|
rule = SafetyRule({"blacklist": ["HARM"]})
|
|
result = await rule.evaluate(_case(), [_turn("this causes harm")])
|
|
assert result.passed is False
|
|
|
|
|
|
async def test_safety_no_blacklist():
|
|
rule = SafetyRule({})
|
|
result = await rule.evaluate(_case(), [_turn("any text")])
|
|
assert result.passed is True
|
|
|
|
|
|
async def test_safety_moderation_api_flagged():
|
|
rule = SafetyRule({
|
|
"use_moderation_api": True,
|
|
"api_url": "http://mock/v1/moderations",
|
|
"api_key": "test",
|
|
})
|
|
mock_response = MagicMock()
|
|
mock_response.raise_for_status = MagicMock()
|
|
mock_response.json = MagicMock(return_value={
|
|
"results": [{"flagged": True, "categories": {"hate": True, "violence": False}}]
|
|
})
|
|
|
|
with patch("agenteval.evaluation.rules.safety.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(return_value=mock_response)
|
|
result = await rule.evaluate(_case(), [_turn("hateful content here")])
|
|
|
|
assert result.passed is False
|
|
assert "hate" in result.reason
|
|
|
|
|
|
async def test_safety_moderation_api_unavailable_degrades():
|
|
rule = SafetyRule({
|
|
"use_moderation_api": True,
|
|
"api_url": "http://unavailable/v1/moderations",
|
|
})
|
|
with patch("agenteval.evaluation.rules.safety.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(side_effect=Exception("connection refused"))
|
|
result = await rule.evaluate(_case(), [_turn("normal text")])
|
|
|
|
assert result.passed is True
|
|
assert "降级" in result.reason
|
|
|
|
|
|
async def test_safety_empty_dialog():
|
|
rule = SafetyRule({})
|
|
result = await rule.evaluate(_case(), [])
|
|
assert result.passed is False
|
|
|
|
|
|
# ── SemanticSimilarityRule ────────────────────────────────────────────────
|
|
|
|
def test_cosine_identical():
|
|
v = [1.0, 0.0, 0.0]
|
|
assert _cosine(v, v) == pytest.approx(1.0)
|
|
|
|
|
|
def test_cosine_orthogonal():
|
|
assert _cosine([1.0, 0.0], [0.0, 1.0]) == pytest.approx(0.0)
|
|
|
|
|
|
def test_cosine_zero_vector():
|
|
assert _cosine([0.0, 0.0], [1.0, 0.0]) == 0.0
|
|
|
|
|
|
async def test_semantic_missing_api_url():
|
|
rule = SemanticSimilarityRule({"reference": "hello world"})
|
|
result = await rule.evaluate(_case(), [_turn("hello")])
|
|
assert result.passed is False
|
|
assert "api_url" in result.reason
|
|
|
|
|
|
async def test_semantic_missing_reference():
|
|
rule = SemanticSimilarityRule({"api_url": "http://mock/embed"})
|
|
result = await rule.evaluate(_case(), [_turn("hello")])
|
|
assert result.passed is False
|
|
assert "reference" in result.reason
|
|
|
|
|
|
async def test_semantic_high_similarity_passes():
|
|
rule = SemanticSimilarityRule({
|
|
"api_url": "http://mock/embed",
|
|
"reference": "hello world",
|
|
"min_score": 0.8,
|
|
})
|
|
vec = [1.0, 0.0, 0.0]
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
mock_resp.json = MagicMock(return_value={"data": [{"embedding": vec}]})
|
|
|
|
with patch("agenteval.evaluation.rules.semantic.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(return_value=mock_resp)
|
|
result = await rule.evaluate(_case(), [_turn("hello world")])
|
|
|
|
assert result.passed is True
|
|
assert result.score == pytest.approx(1.0)
|
|
|
|
|
|
async def test_semantic_low_similarity_fails():
|
|
rule = SemanticSimilarityRule({
|
|
"api_url": "http://mock/embed",
|
|
"reference": "hello world",
|
|
"min_score": 0.9,
|
|
})
|
|
call_count = {"n": 0}
|
|
|
|
async def mock_post(url, **kwargs):
|
|
mock_resp = MagicMock()
|
|
mock_resp.raise_for_status = MagicMock()
|
|
# First call (reply): orthogonal to reference
|
|
vecs = [[1.0, 0.0], [0.0, 1.0]]
|
|
mock_resp.json = MagicMock(return_value={"data": [{"embedding": vecs[call_count["n"]]}]})
|
|
call_count["n"] += 1
|
|
return mock_resp
|
|
|
|
with patch("agenteval.evaluation.rules.semantic.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(side_effect=mock_post)
|
|
result = await rule.evaluate(_case(), [_turn("completely different")])
|
|
|
|
assert result.passed is False
|
|
|
|
|
|
async def test_semantic_api_error_fails_gracefully():
|
|
rule = SemanticSimilarityRule({
|
|
"api_url": "http://mock/embed",
|
|
"reference": "ref",
|
|
})
|
|
with patch("agenteval.evaluation.rules.semantic.httpx.AsyncClient") as MockClient:
|
|
instance = MockClient.return_value.__aenter__.return_value
|
|
instance.post = AsyncMock(side_effect=Exception("timeout"))
|
|
result = await rule.evaluate(_case(), [_turn("reply")])
|
|
|
|
assert result.passed is False
|
|
assert "embedding 调用失败" in result.reason
|
|
|
|
|
|
async def test_semantic_empty_dialog():
|
|
rule = SemanticSimilarityRule({"api_url": "http://x", "reference": "ref"})
|
|
result = await rule.evaluate(_case(), [])
|
|
assert result.passed is False
|
|
|
|
|
|
# ── Rule combination logic (engine integration) ───────────────────────────
|
|
|
|
from agenteval.evaluation.engine import EvalEngine, TimeoutConfig
|
|
from agenteval.models import EvalTarget, PlatformType, RunStatus, Scenario, TargetStatus
|
|
|
|
from tests.unit.mock_channel import MockChannel
|
|
|
|
|
|
def _make_target() -> EvalTarget:
|
|
return EvalTarget(
|
|
id="t-1", name="t", platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
|
channel_type=__import__("agenteval.models", fromlist=["ChannelType"]).ChannelType.TUTU_API,
|
|
channel_config={"base_url": "http://x", "token": "x", "tenant": "t",
|
|
"chat_channel_id": "c", "chat_contact_id": "u"},
|
|
status=TargetStatus.ACTIVE,
|
|
)
|
|
|
|
|
|
def _build_engine(scenario, session) -> EvalEngine:
|
|
engine = EvalEngine(target=_make_target(), scenario=scenario, session=session)
|
|
engine.channel = MockChannel()
|
|
return engine
|
|
|
|
|
|
async def test_rule_logic_all_passes_when_all_pass(db_session):
|
|
scenario = Scenario(id="s1", name="s", cases=[Case(
|
|
id="c1", type=CaseType.SINGLE, messages=["hi"],
|
|
eval_rules=[
|
|
EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]}), # MockChannel replies "echo: q-1"
|
|
EvalRuleConfig(type="response_time", params={"max_ms": 99999}),
|
|
],
|
|
rule_logic=RuleLogic.ALL,
|
|
)])
|
|
engine = _build_engine(scenario, db_session)
|
|
run = await engine.run()
|
|
assert run.status == RunStatus.COMPLETED
|
|
assert run.summary["passed_cases"] == 1
|
|
|
|
|
|
async def test_rule_logic_all_fails_when_one_fails(db_session):
|
|
scenario = Scenario(id="s1", name="s", cases=[Case(
|
|
id="c1", type=CaseType.SINGLE, messages=["hi"],
|
|
eval_rules=[
|
|
EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]}), # passes
|
|
EvalRuleConfig(type="keyword_match", params={"keywords": ["__IMPOSSIBLE__"]}), # fails
|
|
],
|
|
rule_logic=RuleLogic.ALL,
|
|
)])
|
|
engine = _build_engine(scenario, db_session)
|
|
run = await engine.run()
|
|
assert run.status == RunStatus.COMPLETED
|
|
assert run.summary["failed_cases"] == 1
|
|
|
|
|
|
async def test_rule_logic_any_passes_when_one_passes(db_session):
|
|
scenario = Scenario(id="s1", name="s", cases=[Case(
|
|
id="c1", type=CaseType.SINGLE, messages=["hi"],
|
|
eval_rules=[
|
|
EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]}), # passes
|
|
EvalRuleConfig(type="keyword_match", params={"keywords": ["__IMPOSSIBLE__"]}), # fails
|
|
],
|
|
rule_logic=RuleLogic.ANY,
|
|
)])
|
|
engine = _build_engine(scenario, db_session)
|
|
run = await engine.run()
|
|
assert run.status == RunStatus.COMPLETED
|
|
assert run.summary["passed_cases"] == 1
|
|
|
|
|
|
async def test_rule_logic_weighted_passes_above_threshold(db_session):
|
|
scenario = Scenario(id="s1", name="s", cases=[Case(
|
|
id="c1", type=CaseType.SINGLE, messages=["hi"],
|
|
eval_rules=[
|
|
EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]}, weight=0.8), # passes
|
|
EvalRuleConfig(type="keyword_match", params={"keywords": ["__IMPOSSIBLE__"]}, weight=0.2), # fails
|
|
],
|
|
rule_logic=RuleLogic.WEIGHTED,
|
|
rule_pass_threshold=0.6, # weighted score = 0.8/(0.8+0.2)=0.8 >= 0.6 → pass
|
|
)])
|
|
engine = _build_engine(scenario, db_session)
|
|
run = await engine.run()
|
|
assert run.status == RunStatus.COMPLETED
|
|
assert run.summary["passed_cases"] == 1
|
|
|
|
|
|
async def test_rule_logic_weighted_fails_below_threshold(db_session):
|
|
scenario = Scenario(id="s1", name="s", cases=[Case(
|
|
id="c1", type=CaseType.SINGLE, messages=["hi"],
|
|
eval_rules=[
|
|
EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]}, weight=0.2), # passes
|
|
EvalRuleConfig(type="keyword_match", params={"keywords": ["__IMPOSSIBLE__"]}, weight=0.8), # fails
|
|
],
|
|
rule_logic=RuleLogic.WEIGHTED,
|
|
rule_pass_threshold=0.6, # weighted score = 0.2/(0.2+0.8)=0.2 < 0.6 → fail
|
|
)])
|
|
engine = _build_engine(scenario, db_session)
|
|
run = await engine.run()
|
|
assert run.status == RunStatus.COMPLETED
|
|
assert run.summary["failed_cases"] == 1
|