refactor(engine): thin _run_case and _save_rule_results
Three near-identical Turn(...)+save_turn blocks (send-fail / poll-except / happy path) collapse into one _persist_turn helper differing only by the optional fields set. The期望→隐式规则 translation becomes a pure, unit-testable derive_implicit_rules seam (CONTEXT: 期望与规则叠加生效), and the rule_type→ModelPurpose map is hoisted to a module constant.
This commit is contained in:
parent
d411572607
commit
ffc058f951
@ -8,11 +8,13 @@ via an ``asyncio.Event`` cancel token.
|
||||
import asyncio
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from agenteval.channels.base import EvalChannel
|
||||
from agenteval.channels.factory import ChannelFactory
|
||||
from agenteval.config import get_settings
|
||||
from agenteval.evaluation.implicit_rules import derive_implicit_rules
|
||||
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
|
||||
from agenteval.evaluation.rules import RuleResult, get_rule
|
||||
from agenteval.evaluation.run_summary import build_run_summary
|
||||
@ -59,6 +61,14 @@ def _build_send_message(content: str) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
# 需要模型资源的规则类型 → 评测岗位(ModelPurpose);其余规则无需模型
|
||||
RULE_PURPOSE = {
|
||||
"llm_score": ModelPurpose.JUDGE,
|
||||
"semantic_similarity": ModelPurpose.EMBEDDING,
|
||||
"safety": ModelPurpose.MODERATION,
|
||||
}
|
||||
|
||||
|
||||
class EvalEngine:
|
||||
"""Execute evaluation scenarios against targets.
|
||||
|
||||
@ -237,6 +247,36 @@ class EvalEngine:
|
||||
|
||||
# ── case / turn execution ─────────────────────────────────────────
|
||||
|
||||
def _persist_turn(
|
||||
self,
|
||||
run: EvalRun,
|
||||
case: Case,
|
||||
round_index: int,
|
||||
message: str,
|
||||
sent_at: datetime,
|
||||
*,
|
||||
question_msg_id: Optional[str] = None,
|
||||
reply: Optional[dict] = None,
|
||||
received_at: Optional[datetime] = None,
|
||||
latency_ms: Optional[int] = None,
|
||||
) -> Turn:
|
||||
"""Build, persist, and return a Turn. The three call sites (send-fail /
|
||||
poll-except / happy path) differ only in which optional fields are set."""
|
||||
turn = Turn(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run.id,
|
||||
case_id=case.id,
|
||||
round_index=round_index,
|
||||
sent_message=_build_send_message(message),
|
||||
sent_at=sent_at,
|
||||
question_msg_id=question_msg_id,
|
||||
reply=reply,
|
||||
received_at=received_at,
|
||||
latency_ms=latency_ms,
|
||||
)
|
||||
self.result_repo.save_turn(turn)
|
||||
return turn
|
||||
|
||||
async def _run_case(
|
||||
self,
|
||||
run: EvalRun,
|
||||
@ -277,15 +317,7 @@ class EvalEngine:
|
||||
sent_at = utc_now()
|
||||
send_result = await self.channel.send(message)
|
||||
if not send_result.ok:
|
||||
turn = Turn(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run.id,
|
||||
case_id=case.id,
|
||||
round_index=round_index,
|
||||
sent_message=_build_send_message(message),
|
||||
sent_at=sent_at,
|
||||
)
|
||||
self.result_repo.save_turn(turn)
|
||||
turn = self._persist_turn(run, case, round_index, message, sent_at)
|
||||
await self._save_rule_results(run, case, turn, [], progress_callback)
|
||||
await self._emit(
|
||||
progress_callback,
|
||||
@ -305,17 +337,15 @@ class EvalEngine:
|
||||
)
|
||||
except Exception as poll_exc:
|
||||
received_at = utc_now()
|
||||
turn = Turn(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run.id,
|
||||
case_id=case.id,
|
||||
round_index=round_index,
|
||||
sent_message=_build_send_message(message),
|
||||
sent_at=sent_at,
|
||||
turn = self._persist_turn(
|
||||
run,
|
||||
case,
|
||||
round_index,
|
||||
message,
|
||||
sent_at,
|
||||
question_msg_id=send_result.question_msg_id,
|
||||
received_at=received_at,
|
||||
)
|
||||
self.result_repo.save_turn(turn)
|
||||
await self._emit(
|
||||
progress_callback,
|
||||
"turn_error",
|
||||
@ -332,19 +362,17 @@ class EvalEngine:
|
||||
if sent_at and received_at:
|
||||
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
|
||||
|
||||
turn = Turn(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run.id,
|
||||
case_id=case.id,
|
||||
round_index=round_index,
|
||||
sent_message=_build_send_message(message),
|
||||
sent_at=sent_at,
|
||||
turn = self._persist_turn(
|
||||
run,
|
||||
case,
|
||||
round_index,
|
||||
message,
|
||||
sent_at,
|
||||
question_msg_id=send_result.question_msg_id,
|
||||
reply=reply.raw_message if reply else None,
|
||||
received_at=received_at,
|
||||
latency_ms=latency_ms,
|
||||
)
|
||||
self.result_repo.save_turn(turn)
|
||||
dialog.append(turn)
|
||||
|
||||
await self._emit(
|
||||
@ -380,27 +408,7 @@ class EvalEngine:
|
||||
from agenteval.models import EvalRuleConfig
|
||||
|
||||
rules_config: list[EvalRuleConfig] = list(case.eval_rules)
|
||||
|
||||
implicit_config: list[EvalRuleConfig] = []
|
||||
if case.expectations.response_time_max_ms:
|
||||
implicit_config.append(
|
||||
EvalRuleConfig(
|
||||
type="response_time",
|
||||
params={
|
||||
"max_ms": case.expectations.response_time_max_ms,
|
||||
},
|
||||
)
|
||||
)
|
||||
if case.expectations.keywords_include or case.expectations.keywords_exclude:
|
||||
implicit_config.append(
|
||||
EvalRuleConfig(
|
||||
type="keyword_match",
|
||||
params={
|
||||
"keywords": case.expectations.keywords_include,
|
||||
"exclude_keywords": case.expectations.keywords_exclude,
|
||||
},
|
||||
)
|
||||
)
|
||||
implicit_config = derive_implicit_rules(case.expectations)
|
||||
|
||||
all_replied = bool(dialog) and all(t.reply is not None for t in dialog)
|
||||
|
||||
@ -415,11 +423,7 @@ class EvalEngine:
|
||||
|
||||
all_rules = [(cfg, False) for cfg in rules_config] + [(cfg, True) for cfg in implicit_config]
|
||||
for rule_config, is_implicit in all_rules:
|
||||
purpose = {
|
||||
"llm_score": ModelPurpose.JUDGE,
|
||||
"semantic_similarity": ModelPurpose.EMBEDDING,
|
||||
"safety": ModelPurpose.MODERATION,
|
||||
}.get(rule_config.type)
|
||||
purpose = RULE_PURPOSE.get(rule_config.type)
|
||||
try:
|
||||
model_config = self._resolve_model(purpose) if purpose else None
|
||||
rule = get_rule(
|
||||
|
||||
30
backend/agenteval/evaluation/implicit_rules.py
Normal file
30
backend/agenteval/evaluation/implicit_rules.py
Normal file
@ -0,0 +1,30 @@
|
||||
"""Translate a case's期望 (Expectation) into implicit评估规则.
|
||||
|
||||
期望描述"想要什么",评估规则是"怎么判定"——期望派生的隐式规则与显式规则
|
||||
叠加生效(CONTEXT.md)。此翻译是纯逻辑,独立于规则执行与持久化,便于单测。
|
||||
"""
|
||||
|
||||
from agenteval.models import EvalRuleConfig, Expectation
|
||||
|
||||
|
||||
def derive_implicit_rules(expectation: Expectation) -> list[EvalRuleConfig]:
|
||||
"""Build the implicit rule configs a case's expectation implies."""
|
||||
rules: list[EvalRuleConfig] = []
|
||||
if expectation.response_time_max_ms:
|
||||
rules.append(
|
||||
EvalRuleConfig(
|
||||
type="response_time",
|
||||
params={"max_ms": expectation.response_time_max_ms},
|
||||
)
|
||||
)
|
||||
if expectation.keywords_include or expectation.keywords_exclude:
|
||||
rules.append(
|
||||
EvalRuleConfig(
|
||||
type="keyword_match",
|
||||
params={
|
||||
"keywords": expectation.keywords_include,
|
||||
"exclude_keywords": expectation.keywords_exclude,
|
||||
},
|
||||
)
|
||||
)
|
||||
return rules
|
||||
45
tests/unit/test_implicit_rules.py
Normal file
45
tests/unit/test_implicit_rules.py
Normal file
@ -0,0 +1,45 @@
|
||||
"""Unit tests for derive_implicit_rules — 期望→隐式规则的纯翻译."""
|
||||
|
||||
from agenteval.evaluation.implicit_rules import derive_implicit_rules
|
||||
from agenteval.models import Expectation
|
||||
|
||||
|
||||
def test_empty_expectation_yields_no_rules():
|
||||
assert derive_implicit_rules(Expectation()) == []
|
||||
|
||||
|
||||
def test_intent_and_coherence_alone_yield_no_rules():
|
||||
# intent / coherence_min_score 不派生隐式规则(无对应规则类型)
|
||||
exp = Expectation(intent="问诊分流", coherence_min_score=0.8)
|
||||
assert derive_implicit_rules(exp) == []
|
||||
|
||||
|
||||
def test_response_time_yields_response_time_rule():
|
||||
rules = derive_implicit_rules(Expectation(response_time_max_ms=2000))
|
||||
assert len(rules) == 1
|
||||
assert rules[0].type == "response_time"
|
||||
assert rules[0].params == {"max_ms": 2000}
|
||||
|
||||
|
||||
def test_keywords_include_yields_keyword_rule():
|
||||
rules = derive_implicit_rules(Expectation(keywords_include=["挂号", "门诊"]))
|
||||
assert len(rules) == 1
|
||||
assert rules[0].type == "keyword_match"
|
||||
assert rules[0].params == {"keywords": ["挂号", "门诊"], "exclude_keywords": []}
|
||||
|
||||
|
||||
def test_keywords_exclude_alone_yields_keyword_rule():
|
||||
rules = derive_implicit_rules(Expectation(keywords_exclude=["投诉"]))
|
||||
assert len(rules) == 1
|
||||
assert rules[0].type == "keyword_match"
|
||||
assert rules[0].params == {"keywords": [], "exclude_keywords": ["投诉"]}
|
||||
|
||||
|
||||
def test_both_expectations_yield_two_rules_in_order():
|
||||
exp = Expectation(
|
||||
response_time_max_ms=1500,
|
||||
keywords_include=["预约"],
|
||||
keywords_exclude=["取消"],
|
||||
)
|
||||
rules = derive_implicit_rules(exp)
|
||||
assert [r.type for r in rules] == ["response_time", "keyword_match"]
|
||||
Loading…
Reference in New Issue
Block a user