refactor(engine): thin _run_case and _save_rule_results
Three near-identical Turn(...)+save_turn blocks (send-fail / poll-except / happy path) collapse into one _persist_turn helper differing only by the optional fields set. The期望→隐式规则 translation becomes a pure, unit-testable derive_implicit_rules seam (CONTEXT: 期望与规则叠加生效), and the rule_type→ModelPurpose map is hoisted to a module constant.
This commit is contained in:
parent
d411572607
commit
ffc058f951
@ -8,11 +8,13 @@ via an ``asyncio.Event`` cancel token.
|
|||||||
import asyncio
|
import asyncio
|
||||||
import uuid
|
import uuid
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
from typing import Any, Callable, Optional
|
from typing import Any, Callable, Optional
|
||||||
|
|
||||||
from agenteval.channels.base import EvalChannel
|
from agenteval.channels.base import EvalChannel
|
||||||
from agenteval.channels.factory import ChannelFactory
|
from agenteval.channels.factory import ChannelFactory
|
||||||
from agenteval.config import get_settings
|
from agenteval.config import get_settings
|
||||||
|
from agenteval.evaluation.implicit_rules import derive_implicit_rules
|
||||||
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
|
from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome
|
||||||
from agenteval.evaluation.rules import RuleResult, get_rule
|
from agenteval.evaluation.rules import RuleResult, get_rule
|
||||||
from agenteval.evaluation.run_summary import build_run_summary
|
from agenteval.evaluation.run_summary import build_run_summary
|
||||||
@ -59,6 +61,14 @@ def _build_send_message(content: str) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# 需要模型资源的规则类型 → 评测岗位(ModelPurpose);其余规则无需模型
|
||||||
|
RULE_PURPOSE = {
|
||||||
|
"llm_score": ModelPurpose.JUDGE,
|
||||||
|
"semantic_similarity": ModelPurpose.EMBEDDING,
|
||||||
|
"safety": ModelPurpose.MODERATION,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class EvalEngine:
|
class EvalEngine:
|
||||||
"""Execute evaluation scenarios against targets.
|
"""Execute evaluation scenarios against targets.
|
||||||
|
|
||||||
@ -237,6 +247,36 @@ class EvalEngine:
|
|||||||
|
|
||||||
# ── case / turn execution ─────────────────────────────────────────
|
# ── case / turn execution ─────────────────────────────────────────
|
||||||
|
|
||||||
|
def _persist_turn(
|
||||||
|
self,
|
||||||
|
run: EvalRun,
|
||||||
|
case: Case,
|
||||||
|
round_index: int,
|
||||||
|
message: str,
|
||||||
|
sent_at: datetime,
|
||||||
|
*,
|
||||||
|
question_msg_id: Optional[str] = None,
|
||||||
|
reply: Optional[dict] = None,
|
||||||
|
received_at: Optional[datetime] = None,
|
||||||
|
latency_ms: Optional[int] = None,
|
||||||
|
) -> Turn:
|
||||||
|
"""Build, persist, and return a Turn. The three call sites (send-fail /
|
||||||
|
poll-except / happy path) differ only in which optional fields are set."""
|
||||||
|
turn = Turn(
|
||||||
|
id=str(uuid.uuid4()),
|
||||||
|
run_id=run.id,
|
||||||
|
case_id=case.id,
|
||||||
|
round_index=round_index,
|
||||||
|
sent_message=_build_send_message(message),
|
||||||
|
sent_at=sent_at,
|
||||||
|
question_msg_id=question_msg_id,
|
||||||
|
reply=reply,
|
||||||
|
received_at=received_at,
|
||||||
|
latency_ms=latency_ms,
|
||||||
|
)
|
||||||
|
self.result_repo.save_turn(turn)
|
||||||
|
return turn
|
||||||
|
|
||||||
async def _run_case(
|
async def _run_case(
|
||||||
self,
|
self,
|
||||||
run: EvalRun,
|
run: EvalRun,
|
||||||
@ -277,15 +317,7 @@ class EvalEngine:
|
|||||||
sent_at = utc_now()
|
sent_at = utc_now()
|
||||||
send_result = await self.channel.send(message)
|
send_result = await self.channel.send(message)
|
||||||
if not send_result.ok:
|
if not send_result.ok:
|
||||||
turn = Turn(
|
turn = self._persist_turn(run, case, round_index, message, sent_at)
|
||||||
id=str(uuid.uuid4()),
|
|
||||||
run_id=run.id,
|
|
||||||
case_id=case.id,
|
|
||||||
round_index=round_index,
|
|
||||||
sent_message=_build_send_message(message),
|
|
||||||
sent_at=sent_at,
|
|
||||||
)
|
|
||||||
self.result_repo.save_turn(turn)
|
|
||||||
await self._save_rule_results(run, case, turn, [], progress_callback)
|
await self._save_rule_results(run, case, turn, [], progress_callback)
|
||||||
await self._emit(
|
await self._emit(
|
||||||
progress_callback,
|
progress_callback,
|
||||||
@ -305,17 +337,15 @@ class EvalEngine:
|
|||||||
)
|
)
|
||||||
except Exception as poll_exc:
|
except Exception as poll_exc:
|
||||||
received_at = utc_now()
|
received_at = utc_now()
|
||||||
turn = Turn(
|
turn = self._persist_turn(
|
||||||
id=str(uuid.uuid4()),
|
run,
|
||||||
run_id=run.id,
|
case,
|
||||||
case_id=case.id,
|
round_index,
|
||||||
round_index=round_index,
|
message,
|
||||||
sent_message=_build_send_message(message),
|
sent_at,
|
||||||
sent_at=sent_at,
|
|
||||||
question_msg_id=send_result.question_msg_id,
|
question_msg_id=send_result.question_msg_id,
|
||||||
received_at=received_at,
|
received_at=received_at,
|
||||||
)
|
)
|
||||||
self.result_repo.save_turn(turn)
|
|
||||||
await self._emit(
|
await self._emit(
|
||||||
progress_callback,
|
progress_callback,
|
||||||
"turn_error",
|
"turn_error",
|
||||||
@ -332,19 +362,17 @@ class EvalEngine:
|
|||||||
if sent_at and received_at:
|
if sent_at and received_at:
|
||||||
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
|
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
|
||||||
|
|
||||||
turn = Turn(
|
turn = self._persist_turn(
|
||||||
id=str(uuid.uuid4()),
|
run,
|
||||||
run_id=run.id,
|
case,
|
||||||
case_id=case.id,
|
round_index,
|
||||||
round_index=round_index,
|
message,
|
||||||
sent_message=_build_send_message(message),
|
sent_at,
|
||||||
sent_at=sent_at,
|
|
||||||
question_msg_id=send_result.question_msg_id,
|
question_msg_id=send_result.question_msg_id,
|
||||||
reply=reply.raw_message if reply else None,
|
reply=reply.raw_message if reply else None,
|
||||||
received_at=received_at,
|
received_at=received_at,
|
||||||
latency_ms=latency_ms,
|
latency_ms=latency_ms,
|
||||||
)
|
)
|
||||||
self.result_repo.save_turn(turn)
|
|
||||||
dialog.append(turn)
|
dialog.append(turn)
|
||||||
|
|
||||||
await self._emit(
|
await self._emit(
|
||||||
@ -380,27 +408,7 @@ class EvalEngine:
|
|||||||
from agenteval.models import EvalRuleConfig
|
from agenteval.models import EvalRuleConfig
|
||||||
|
|
||||||
rules_config: list[EvalRuleConfig] = list(case.eval_rules)
|
rules_config: list[EvalRuleConfig] = list(case.eval_rules)
|
||||||
|
implicit_config = derive_implicit_rules(case.expectations)
|
||||||
implicit_config: list[EvalRuleConfig] = []
|
|
||||||
if case.expectations.response_time_max_ms:
|
|
||||||
implicit_config.append(
|
|
||||||
EvalRuleConfig(
|
|
||||||
type="response_time",
|
|
||||||
params={
|
|
||||||
"max_ms": case.expectations.response_time_max_ms,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if case.expectations.keywords_include or case.expectations.keywords_exclude:
|
|
||||||
implicit_config.append(
|
|
||||||
EvalRuleConfig(
|
|
||||||
type="keyword_match",
|
|
||||||
params={
|
|
||||||
"keywords": case.expectations.keywords_include,
|
|
||||||
"exclude_keywords": case.expectations.keywords_exclude,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
all_replied = bool(dialog) and all(t.reply is not None for t in dialog)
|
all_replied = bool(dialog) and all(t.reply is not None for t in dialog)
|
||||||
|
|
||||||
@ -415,11 +423,7 @@ class EvalEngine:
|
|||||||
|
|
||||||
all_rules = [(cfg, False) for cfg in rules_config] + [(cfg, True) for cfg in implicit_config]
|
all_rules = [(cfg, False) for cfg in rules_config] + [(cfg, True) for cfg in implicit_config]
|
||||||
for rule_config, is_implicit in all_rules:
|
for rule_config, is_implicit in all_rules:
|
||||||
purpose = {
|
purpose = RULE_PURPOSE.get(rule_config.type)
|
||||||
"llm_score": ModelPurpose.JUDGE,
|
|
||||||
"semantic_similarity": ModelPurpose.EMBEDDING,
|
|
||||||
"safety": ModelPurpose.MODERATION,
|
|
||||||
}.get(rule_config.type)
|
|
||||||
try:
|
try:
|
||||||
model_config = self._resolve_model(purpose) if purpose else None
|
model_config = self._resolve_model(purpose) if purpose else None
|
||||||
rule = get_rule(
|
rule = get_rule(
|
||||||
|
|||||||
30
backend/agenteval/evaluation/implicit_rules.py
Normal file
30
backend/agenteval/evaluation/implicit_rules.py
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
"""Translate a case's期望 (Expectation) into implicit评估规则.
|
||||||
|
|
||||||
|
期望描述"想要什么",评估规则是"怎么判定"——期望派生的隐式规则与显式规则
|
||||||
|
叠加生效(CONTEXT.md)。此翻译是纯逻辑,独立于规则执行与持久化,便于单测。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from agenteval.models import EvalRuleConfig, Expectation
|
||||||
|
|
||||||
|
|
||||||
|
def derive_implicit_rules(expectation: Expectation) -> list[EvalRuleConfig]:
|
||||||
|
"""Build the implicit rule configs a case's expectation implies."""
|
||||||
|
rules: list[EvalRuleConfig] = []
|
||||||
|
if expectation.response_time_max_ms:
|
||||||
|
rules.append(
|
||||||
|
EvalRuleConfig(
|
||||||
|
type="response_time",
|
||||||
|
params={"max_ms": expectation.response_time_max_ms},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if expectation.keywords_include or expectation.keywords_exclude:
|
||||||
|
rules.append(
|
||||||
|
EvalRuleConfig(
|
||||||
|
type="keyword_match",
|
||||||
|
params={
|
||||||
|
"keywords": expectation.keywords_include,
|
||||||
|
"exclude_keywords": expectation.keywords_exclude,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return rules
|
||||||
45
tests/unit/test_implicit_rules.py
Normal file
45
tests/unit/test_implicit_rules.py
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
"""Unit tests for derive_implicit_rules — 期望→隐式规则的纯翻译."""
|
||||||
|
|
||||||
|
from agenteval.evaluation.implicit_rules import derive_implicit_rules
|
||||||
|
from agenteval.models import Expectation
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_expectation_yields_no_rules():
|
||||||
|
assert derive_implicit_rules(Expectation()) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_intent_and_coherence_alone_yield_no_rules():
|
||||||
|
# intent / coherence_min_score 不派生隐式规则(无对应规则类型)
|
||||||
|
exp = Expectation(intent="问诊分流", coherence_min_score=0.8)
|
||||||
|
assert derive_implicit_rules(exp) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_response_time_yields_response_time_rule():
|
||||||
|
rules = derive_implicit_rules(Expectation(response_time_max_ms=2000))
|
||||||
|
assert len(rules) == 1
|
||||||
|
assert rules[0].type == "response_time"
|
||||||
|
assert rules[0].params == {"max_ms": 2000}
|
||||||
|
|
||||||
|
|
||||||
|
def test_keywords_include_yields_keyword_rule():
|
||||||
|
rules = derive_implicit_rules(Expectation(keywords_include=["挂号", "门诊"]))
|
||||||
|
assert len(rules) == 1
|
||||||
|
assert rules[0].type == "keyword_match"
|
||||||
|
assert rules[0].params == {"keywords": ["挂号", "门诊"], "exclude_keywords": []}
|
||||||
|
|
||||||
|
|
||||||
|
def test_keywords_exclude_alone_yields_keyword_rule():
|
||||||
|
rules = derive_implicit_rules(Expectation(keywords_exclude=["投诉"]))
|
||||||
|
assert len(rules) == 1
|
||||||
|
assert rules[0].type == "keyword_match"
|
||||||
|
assert rules[0].params == {"keywords": [], "exclude_keywords": ["投诉"]}
|
||||||
|
|
||||||
|
|
||||||
|
def test_both_expectations_yield_two_rules_in_order():
|
||||||
|
exp = Expectation(
|
||||||
|
response_time_max_ms=1500,
|
||||||
|
keywords_include=["预约"],
|
||||||
|
keywords_exclude=["取消"],
|
||||||
|
)
|
||||||
|
rules = derive_implicit_rules(exp)
|
||||||
|
assert [r.type for r in rules] == ["response_time", "keyword_match"]
|
||||||
Loading…
Reference in New Issue
Block a user