refactor(engine): thin _run_case and _save_rule_results

Three near-identical Turn(...)+save_turn blocks (send-fail / poll-except /
happy path) collapse into one _persist_turn helper differing only by the
optional fields set. The期望→隐式规则 translation becomes a pure,
unit-testable derive_implicit_rules seam (CONTEXT: 期望与规则叠加生效),
and the rule_type→ModelPurpose map is hoisted to a module constant.
This commit is contained in:
sinohqb 2026-07-31 14:59:50 +08:00
parent d411572607
commit ffc058f951
3 changed files with 130 additions and 51 deletions

View File

@ -8,11 +8,13 @@ via an ``asyncio.Event`` cancel token.
import asyncio import asyncio
import uuid import uuid
from dataclasses import dataclass from dataclasses import dataclass
from datetime import datetime
from typing import Any, Callable, Optional from typing import Any, Callable, Optional
from agenteval.channels.base import EvalChannel from agenteval.channels.base import EvalChannel
from agenteval.channels.factory import ChannelFactory from agenteval.channels.factory import ChannelFactory
from agenteval.config import get_settings from agenteval.config import get_settings
from agenteval.evaluation.implicit_rules import derive_implicit_rules
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
from agenteval.evaluation.rules import RuleResult, get_rule from agenteval.evaluation.rules import RuleResult, get_rule
from agenteval.evaluation.run_summary import build_run_summary from agenteval.evaluation.run_summary import build_run_summary
@ -59,6 +61,14 @@ def _build_send_message(content: str) -> dict[str, Any]:
} }
# 需要模型资源的规则类型 → 评测岗位ModelPurpose其余规则无需模型
RULE_PURPOSE = {
"llm_score": ModelPurpose.JUDGE,
"semantic_similarity": ModelPurpose.EMBEDDING,
"safety": ModelPurpose.MODERATION,
}
class EvalEngine: class EvalEngine:
"""Execute evaluation scenarios against targets. """Execute evaluation scenarios against targets.
@ -237,6 +247,36 @@ class EvalEngine:
# ── case / turn execution ───────────────────────────────────────── # ── case / turn execution ─────────────────────────────────────────
def _persist_turn(
self,
run: EvalRun,
case: Case,
round_index: int,
message: str,
sent_at: datetime,
*,
question_msg_id: Optional[str] = None,
reply: Optional[dict] = None,
received_at: Optional[datetime] = None,
latency_ms: Optional[int] = None,
) -> Turn:
"""Build, persist, and return a Turn. The three call sites (send-fail /
poll-except / happy path) differ only in which optional fields are set."""
turn = Turn(
id=str(uuid.uuid4()),
run_id=run.id,
case_id=case.id,
round_index=round_index,
sent_message=_build_send_message(message),
sent_at=sent_at,
question_msg_id=question_msg_id,
reply=reply,
received_at=received_at,
latency_ms=latency_ms,
)
self.result_repo.save_turn(turn)
return turn
async def _run_case( async def _run_case(
self, self,
run: EvalRun, run: EvalRun,
@ -277,15 +317,7 @@ class EvalEngine:
sent_at = utc_now() sent_at = utc_now()
send_result = await self.channel.send(message) send_result = await self.channel.send(message)
if not send_result.ok: if not send_result.ok:
turn = Turn( turn = self._persist_turn(run, case, round_index, message, sent_at)
id=str(uuid.uuid4()),
run_id=run.id,
case_id=case.id,
round_index=round_index,
sent_message=_build_send_message(message),
sent_at=sent_at,
)
self.result_repo.save_turn(turn)
await self._save_rule_results(run, case, turn, [], progress_callback) await self._save_rule_results(run, case, turn, [], progress_callback)
await self._emit( await self._emit(
progress_callback, progress_callback,
@ -305,17 +337,15 @@ class EvalEngine:
) )
except Exception as poll_exc: except Exception as poll_exc:
received_at = utc_now() received_at = utc_now()
turn = Turn( turn = self._persist_turn(
id=str(uuid.uuid4()), run,
run_id=run.id, case,
case_id=case.id, round_index,
round_index=round_index, message,
sent_message=_build_send_message(message), sent_at,
sent_at=sent_at,
question_msg_id=send_result.question_msg_id, question_msg_id=send_result.question_msg_id,
received_at=received_at, received_at=received_at,
) )
self.result_repo.save_turn(turn)
await self._emit( await self._emit(
progress_callback, progress_callback,
"turn_error", "turn_error",
@ -332,19 +362,17 @@ class EvalEngine:
if sent_at and received_at: if sent_at and received_at:
latency_ms = int((received_at - sent_at).total_seconds() * 1000) latency_ms = int((received_at - sent_at).total_seconds() * 1000)
turn = Turn( turn = self._persist_turn(
id=str(uuid.uuid4()), run,
run_id=run.id, case,
case_id=case.id, round_index,
round_index=round_index, message,
sent_message=_build_send_message(message), sent_at,
sent_at=sent_at,
question_msg_id=send_result.question_msg_id, question_msg_id=send_result.question_msg_id,
reply=reply.raw_message if reply else None, reply=reply.raw_message if reply else None,
received_at=received_at, received_at=received_at,
latency_ms=latency_ms, latency_ms=latency_ms,
) )
self.result_repo.save_turn(turn)
dialog.append(turn) dialog.append(turn)
await self._emit( await self._emit(
@ -380,27 +408,7 @@ class EvalEngine:
from agenteval.models import EvalRuleConfig from agenteval.models import EvalRuleConfig
rules_config: list[EvalRuleConfig] = list(case.eval_rules) rules_config: list[EvalRuleConfig] = list(case.eval_rules)
implicit_config = derive_implicit_rules(case.expectations)
implicit_config: list[EvalRuleConfig] = []
if case.expectations.response_time_max_ms:
implicit_config.append(
EvalRuleConfig(
type="response_time",
params={
"max_ms": case.expectations.response_time_max_ms,
},
)
)
if case.expectations.keywords_include or case.expectations.keywords_exclude:
implicit_config.append(
EvalRuleConfig(
type="keyword_match",
params={
"keywords": case.expectations.keywords_include,
"exclude_keywords": case.expectations.keywords_exclude,
},
)
)
all_replied = bool(dialog) and all(t.reply is not None for t in dialog) all_replied = bool(dialog) and all(t.reply is not None for t in dialog)
@ -415,11 +423,7 @@ class EvalEngine:
all_rules = [(cfg, False) for cfg in rules_config] + [(cfg, True) for cfg in implicit_config] all_rules = [(cfg, False) for cfg in rules_config] + [(cfg, True) for cfg in implicit_config]
for rule_config, is_implicit in all_rules: for rule_config, is_implicit in all_rules:
purpose = { purpose = RULE_PURPOSE.get(rule_config.type)
"llm_score": ModelPurpose.JUDGE,
"semantic_similarity": ModelPurpose.EMBEDDING,
"safety": ModelPurpose.MODERATION,
}.get(rule_config.type)
try: try:
model_config = self._resolve_model(purpose) if purpose else None model_config = self._resolve_model(purpose) if purpose else None
rule = get_rule( rule = get_rule(

View File

@ -0,0 +1,30 @@
"""Translate a case's期望 (Expectation) into implicit评估规则.
期望描述"想要什么"评估规则是"怎么判定"期望派生的隐式规则与显式规则
叠加生效CONTEXT.md此翻译是纯逻辑独立于规则执行与持久化便于单测
"""
from agenteval.models import EvalRuleConfig, Expectation
def derive_implicit_rules(expectation: Expectation) -> list[EvalRuleConfig]:
"""Build the implicit rule configs a case's expectation implies."""
rules: list[EvalRuleConfig] = []
if expectation.response_time_max_ms:
rules.append(
EvalRuleConfig(
type="response_time",
params={"max_ms": expectation.response_time_max_ms},
)
)
if expectation.keywords_include or expectation.keywords_exclude:
rules.append(
EvalRuleConfig(
type="keyword_match",
params={
"keywords": expectation.keywords_include,
"exclude_keywords": expectation.keywords_exclude,
},
)
)
return rules

View File

@ -0,0 +1,45 @@
"""Unit tests for derive_implicit_rules — 期望→隐式规则的纯翻译."""
from agenteval.evaluation.implicit_rules import derive_implicit_rules
from agenteval.models import Expectation
def test_empty_expectation_yields_no_rules():
assert derive_implicit_rules(Expectation()) == []
def test_intent_and_coherence_alone_yield_no_rules():
# intent / coherence_min_score 不派生隐式规则(无对应规则类型)
exp = Expectation(intent="问诊分流", coherence_min_score=0.8)
assert derive_implicit_rules(exp) == []
def test_response_time_yields_response_time_rule():
rules = derive_implicit_rules(Expectation(response_time_max_ms=2000))
assert len(rules) == 1
assert rules[0].type == "response_time"
assert rules[0].params == {"max_ms": 2000}
def test_keywords_include_yields_keyword_rule():
rules = derive_implicit_rules(Expectation(keywords_include=["挂号", "门诊"]))
assert len(rules) == 1
assert rules[0].type == "keyword_match"
assert rules[0].params == {"keywords": ["挂号", "门诊"], "exclude_keywords": []}
def test_keywords_exclude_alone_yields_keyword_rule():
rules = derive_implicit_rules(Expectation(keywords_exclude=["投诉"]))
assert len(rules) == 1
assert rules[0].type == "keyword_match"
assert rules[0].params == {"keywords": [], "exclude_keywords": ["投诉"]}
def test_both_expectations_yield_two_rules_in_order():
exp = Expectation(
response_time_max_ms=1500,
keywords_include=["预约"],
keywords_exclude=["取消"],
)
rules = derive_implicit_rules(exp)
assert [r.type for r in rules] == ["response_time", "keyword_match"]