2f09ee2bfc
All checks were successful
CI / test (pull_request) Successful in 3m58s
- 报告渲染 Go/No-Go 上线评估横幅（HTML 彩色 banner + Markdown 引用块） - 抽取 scored_llm 共享模块：llm_score / fluency 直连调用与评分解析收敛 - 网关新增 chat_with_usage / embed_with_usage，规则按次归集 llm_usage - 引擎分岗位用量归集（judge/generator/embedding/moderation）写入 RunSummary.eval_usage_by_purpose，并发下不做总量差值 - cost_tracking 重构：data/model_pricing.json 覆盖 + 默认计价表， 删除从未有数据支撑的 Turn 维度成本函数（偏差说明见 PR） - 报告 summary 增加 eval_cost 分岗位成本段并在 Markdown 渲染
136 lines
5.3 KiB
Python
136 lines
5.3 KiB
Python
"""Fluency assessment rule using LLM to evaluate conversation naturalness."""
|
||
|
||
import json
|
||
|
||
from agenteval.evaluation.rules.base import EvalRule, RuleResult, register_rule
|
||
from agenteval.evaluation.rules.scored_llm import call_scored_llm, parse_scored_content
|
||
from agenteval.models import Case, Turn
|
||
from agenteval.utils.llm import extract_reply_text
|
||
|
||
|
||
@register_rule
|
||
class FluencyRule(EvalRule):
|
||
"""Use LLM to evaluate conversation fluency and naturalness.
|
||
|
||
Evaluates:
|
||
- Naturalness: Does the conversation flow naturally?
|
||
- Repetition: Are there unnecessary repetitions?
|
||
- Coherence: Is the conversation coherent and logical?
|
||
|
||
Configuration params:
|
||
min_score: Minimum fluency score to pass (0-10, default: 7)
|
||
criteria: Custom evaluation criteria (optional)
|
||
"""
|
||
|
||
name = "fluency"
|
||
|
||
async def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult:
|
||
if not dialog:
|
||
return RuleResult(passed=False, reason="无回复记录")
|
||
|
||
# Build conversation context
|
||
conversation_parts = []
|
||
for turn in dialog:
|
||
# Extract sent message
|
||
sent_text = ""
|
||
if turn.sent_message:
|
||
sent_text = self._extract_message_text(turn.sent_message)
|
||
|
||
# Extract reply
|
||
reply_text = extract_reply_text(turn.reply) if turn.reply else ""
|
||
|
||
if sent_text:
|
||
conversation_parts.append(f"用户: {sent_text}")
|
||
if reply_text:
|
||
conversation_parts.append(f"助手: {reply_text}")
|
||
|
||
if not conversation_parts:
|
||
return RuleResult(passed=False, reason="无法提取对话内容")
|
||
|
||
conversation_text = "\n".join(conversation_parts)
|
||
|
||
# Get evaluation criteria
|
||
min_score = float(self.params.get("min_score", 7))
|
||
custom_criteria = self.params.get("criteria", "")
|
||
|
||
# Call LLM for fluency assessment
|
||
if self.model_config and self.gateway:
|
||
score, reason = await self._evaluate_fluency(conversation_text, custom_criteria)
|
||
else:
|
||
api_url = self.params.get("api_url")
|
||
api_key = self.params.get("api_key")
|
||
model = self.params.get("model", "gpt-4o-mini")
|
||
if not api_url:
|
||
return RuleResult(passed=False, reason="流畅度评估规则未绑定评估模型")
|
||
score, reason = await self._call_llm(api_url, api_key, model, conversation_text, custom_criteria)
|
||
|
||
if score is None:
|
||
return RuleResult(passed=False, reason=f"流畅度评估失败: {reason}")
|
||
|
||
passed = score >= min_score
|
||
verdict = "通过" if passed else "未通过"
|
||
return RuleResult(
|
||
passed=passed,
|
||
score=score / 10.0,
|
||
reason=f"流畅度评分 {score}/10，{verdict} (阈值 {min_score})；{reason}",
|
||
)
|
||
|
||
def _extract_message_text(self, sent_message: dict) -> str:
|
||
"""Extract text from sent_message dict."""
|
||
body = sent_message.get("msgBody", "")
|
||
if isinstance(body, dict):
|
||
return body.get("content", "")
|
||
try:
|
||
return json.loads(body).get("content", "")
|
||
except Exception:
|
||
return str(body)
|
||
|
||
async def _evaluate_fluency(self, conversation: str, custom_criteria: str) -> tuple[float | None, str]:
|
||
"""Evaluate conversation fluency using the gateway."""
|
||
system_prompt, user_prompt = self._build_prompts(conversation, custom_criteria)
|
||
try:
|
||
content, usage = await self.gateway.chat_with_usage(
|
||
self.model_config,
|
||
[
|
||
{"role": "system", "content": system_prompt},
|
||
{"role": "user", "content": user_prompt},
|
||
],
|
||
temperature=0.2,
|
||
)
|
||
self._record_llm_usage(usage)
|
||
return parse_scored_content(content)
|
||
except Exception as exc:
|
||
return None, str(exc)
|
||
|
||
async def _call_llm(
|
||
self,
|
||
api_url: str,
|
||
api_key: str | None,
|
||
model: str,
|
||
conversation: str,
|
||
custom_criteria: str,
|
||
) -> tuple[float | None, str]:
|
||
"""Call LLM API directly for fluency assessment."""
|
||
system_prompt, user_prompt = self._build_prompts(conversation, custom_criteria)
|
||
return await call_scored_llm(api_url, api_key, model, system_prompt, user_prompt)
|
||
|
||
@staticmethod
|
||
def _build_prompts(conversation: str, custom_criteria: str) -> tuple[str, str]:
|
||
"""Build system and user prompts for fluency evaluation."""
|
||
default_criteria = """评估对话的流畅度和自然性，考虑以下方面：
|
||
1. 自然性：对话是否流畅自然，像真人对话？
|
||
2. 重复性：是否有不必要的重复或冗余？
|
||
3. 连贯性：对话是否逻辑连贯，上下文一致？
|
||
4. 响应质量：助手的回复是否恰当、有帮助？"""
|
||
|
||
criteria = custom_criteria if custom_criteria else default_criteria
|
||
|
||
system_prompt = (
|
||
"你是一位对话质量评估专家。请评估以下对话的流畅度和自然性。\n"
|
||
f"评估标准：{criteria}\n"
|
||
"打分范围：0-10分（10分最高）\n"
|
||
'只输出一个 JSON 对象：{"score": number, "reason": "简短说明"}'
|
||
)
|
||
user_prompt = f"对话内容：\n{conversation}"
|
||
return system_prompt, user_prompt
|