21cc6ed407
All checks were successful
CI / test (pull_request) Successful in 4m11s
新增用户体验跟踪能力。 - RunSummary 新增 abandoned_cases 和 abandonment_rate 字段 - 新增 fluency 规则：LLM 评估对话流畅度和自然性（0-10 分） - 流畅度评估考虑：自然性、重复性、连贯性、响应质量 - 新增 10 项单元测试（676 tests passed） 注：放弃率的实际计算逻辑需要在引擎中集成，本 PR 提供数据模型和规则基础设施。 Closes #26
111 lines
3.4 KiB
Python
111 lines
3.4 KiB
Python
"""Tests for fluency assessment rule and abandonment tracking."""
|
|
|
|
import pytest
|
|
|
|
from agenteval.evaluation.rules.base import get_rule
|
|
from agenteval.models import Case, RunSummary, Turn
|
|
|
|
|
|
def test_run_summary_abandonment_fields():
|
|
"""RunSummary should have abandonment tracking fields."""
|
|
summary = RunSummary(
|
|
total_cases=10,
|
|
passed_cases=7,
|
|
failed_cases=2,
|
|
abandoned_cases=1,
|
|
)
|
|
assert summary.total_cases == 10
|
|
assert summary.abandoned_cases == 1
|
|
assert summary.abandonment_rate is None # Not calculated yet
|
|
|
|
|
|
def test_run_summary_abandonment_rate_calculation():
|
|
"""Abandonment rate should be calculable from summary fields."""
|
|
summary = RunSummary(
|
|
total_cases=10,
|
|
abandoned_cases=2,
|
|
)
|
|
# Calculate abandonment rate
|
|
if summary.total_cases > 0:
|
|
rate = summary.abandoned_cases / summary.total_cases
|
|
assert rate == 0.2
|
|
|
|
|
|
def test_fluency_rule_registered():
|
|
"""Fluency rule should be registered in the rule registry."""
|
|
from agenteval.evaluation.rules import list_rule_types
|
|
|
|
assert "fluency" in list_rule_types()
|
|
|
|
|
|
def test_fluency_rule_config():
|
|
"""Fluency rule should accept configuration parameters."""
|
|
rule = get_rule(
|
|
"fluency",
|
|
{"min_score": 8, "criteria": "评估对话是否自然流畅"},
|
|
)
|
|
assert rule.params["min_score"] == 8
|
|
assert rule.params["criteria"] == "评估对话是否自然流畅"
|
|
|
|
|
|
def test_fluency_rule_empty_dialog():
|
|
"""Fluency rule should fail on empty dialog."""
|
|
rule = get_rule("fluency", {"min_score": 7})
|
|
case = Case(id="c1", messages=["hello"])
|
|
import asyncio
|
|
|
|
result = asyncio.run(rule.evaluate(case, []))
|
|
assert result.passed is False
|
|
assert "无回复记录" in result.reason
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fluency_rule_no_model_config():
|
|
"""Fluency rule should fail without model configuration."""
|
|
rule = get_rule("fluency", {"min_score": 7})
|
|
case = Case(id="c1", messages=["hello"])
|
|
dialog = [
|
|
Turn(
|
|
id="t1",
|
|
run_id="r1",
|
|
case_id="c1",
|
|
round_index=1,
|
|
sent_message={"msgBody": "你好"},
|
|
reply={"msgBody": "您好，有什么可以帮助您的？"},
|
|
)
|
|
]
|
|
result = await rule.evaluate(case, dialog)
|
|
assert result.passed is False
|
|
assert "未绑定评估模型" in result.reason
|
|
|
|
|
|
def test_fluency_rule_default_min_score():
|
|
"""Fluency rule should have default min_score of 7."""
|
|
rule = get_rule("fluency", {})
|
|
assert rule.params.get("min_score", 7) == 7
|
|
|
|
|
|
def test_abandonment_rate_zero_total():
|
|
"""Abandonment rate should handle zero total cases."""
|
|
summary = RunSummary(total_cases=0, abandoned_cases=0)
|
|
# Should not raise division by zero
|
|
if summary.total_cases > 0:
|
|
rate = summary.abandoned_cases / summary.total_cases
|
|
else:
|
|
rate = None
|
|
assert rate is None
|
|
|
|
|
|
def test_abandonment_rate_all_abandoned():
|
|
"""Abandonment rate should be 1.0 when all cases are abandoned."""
|
|
summary = RunSummary(total_cases=5, abandoned_cases=5)
|
|
rate = summary.abandoned_cases / summary.total_cases
|
|
assert rate == 1.0
|
|
|
|
|
|
def test_abandonment_rate_none_abandoned():
|
|
"""Abandonment rate should be 0.0 when no cases are abandoned."""
|
|
summary = RunSummary(total_cases=5, abandoned_cases=0)
|
|
rate = summary.abandoned_cases / summary.total_cases
|
|
assert rate == 0.0
|