From ffc058f95166a8eb888c017400b78617105ba056 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Fri, 31 Jul 2026 14:59:50 +0800 Subject: [PATCH] refactor(engine): thin _run_case and _save_rule_results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three near-identical Turn(...)+save_turn blocks (send-fail / poll-except / happy path) collapse into one _persist_turn helper differing only by the optional fields set. The期望→隐式规则 translation becomes a pure, unit-testable derive_implicit_rules seam (CONTEXT: 期望与规则叠加生效), and the rule_type→ModelPurpose map is hoisted to a module constant. --- backend/agenteval/evaluation/engine.py | 106 +++++++++--------- .../agenteval/evaluation/implicit_rules.py | 30 +++++ tests/unit/test_implicit_rules.py | 45 ++++++++ 3 files changed, 130 insertions(+), 51 deletions(-) create mode 100644 backend/agenteval/evaluation/implicit_rules.py create mode 100644 tests/unit/test_implicit_rules.py diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index 0fb94e2..87918a4 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -8,11 +8,13 @@ via an ``asyncio.Event`` cancel token. import asyncio import uuid from dataclasses import dataclass +from datetime import datetime from typing import Any, Callable, Optional from agenteval.channels.base import EvalChannel from agenteval.channels.factory import ChannelFactory from agenteval.config import get_settings +from agenteval.evaluation.implicit_rules import derive_implicit_rules from agenteval.evaluation.judgement import CaseOutcome, RuleOutcome, combine_case_outcome from agenteval.evaluation.rules import RuleResult, get_rule from agenteval.evaluation.run_summary import build_run_summary @@ -59,6 +61,14 @@ def _build_send_message(content: str) -> dict[str, Any]: } +# 需要模型资源的规则类型 → 评测岗位（ModelPurpose）；其余规则无需模型 +RULE_PURPOSE = { + "llm_score": ModelPurpose.JUDGE, + "semantic_similarity": ModelPurpose.EMBEDDING, + "safety": ModelPurpose.MODERATION, +} + + class EvalEngine: """Execute evaluation scenarios against targets. @@ -237,6 +247,36 @@ class EvalEngine: # ── case / turn execution ───────────────────────────────────────── + def _persist_turn( + self, + run: EvalRun, + case: Case, + round_index: int, + message: str, + sent_at: datetime, + *, + question_msg_id: Optional[str] = None, + reply: Optional[dict] = None, + received_at: Optional[datetime] = None, + latency_ms: Optional[int] = None, + ) -> Turn: + """Build, persist, and return a Turn. The three call sites (send-fail / + poll-except / happy path) differ only in which optional fields are set.""" + turn = Turn( + id=str(uuid.uuid4()), + run_id=run.id, + case_id=case.id, + round_index=round_index, + sent_message=_build_send_message(message), + sent_at=sent_at, + question_msg_id=question_msg_id, + reply=reply, + received_at=received_at, + latency_ms=latency_ms, + ) + self.result_repo.save_turn(turn) + return turn + async def _run_case( self, run: EvalRun, @@ -277,15 +317,7 @@ class EvalEngine: sent_at = utc_now() send_result = await self.channel.send(message) if not send_result.ok: - turn = Turn( - id=str(uuid.uuid4()), - run_id=run.id, - case_id=case.id, - round_index=round_index, - sent_message=_build_send_message(message), - sent_at=sent_at, - ) - self.result_repo.save_turn(turn) + turn = self._persist_turn(run, case, round_index, message, sent_at) await self._save_rule_results(run, case, turn, [], progress_callback) await self._emit( progress_callback, @@ -305,17 +337,15 @@ class EvalEngine: ) except Exception as poll_exc: received_at = utc_now() - turn = Turn( - id=str(uuid.uuid4()), - run_id=run.id, - case_id=case.id, - round_index=round_index, - sent_message=_build_send_message(message), - sent_at=sent_at, + turn = self._persist_turn( + run, + case, + round_index, + message, + sent_at, question_msg_id=send_result.question_msg_id, received_at=received_at, ) - self.result_repo.save_turn(turn) await self._emit( progress_callback, "turn_error", @@ -332,19 +362,17 @@ class EvalEngine: if sent_at and received_at: latency_ms = int((received_at - sent_at).total_seconds() * 1000) - turn = Turn( - id=str(uuid.uuid4()), - run_id=run.id, - case_id=case.id, - round_index=round_index, - sent_message=_build_send_message(message), - sent_at=sent_at, + turn = self._persist_turn( + run, + case, + round_index, + message, + sent_at, question_msg_id=send_result.question_msg_id, reply=reply.raw_message if reply else None, received_at=received_at, latency_ms=latency_ms, ) - self.result_repo.save_turn(turn) dialog.append(turn) await self._emit( @@ -380,27 +408,7 @@ class EvalEngine: from agenteval.models import EvalRuleConfig rules_config: list[EvalRuleConfig] = list(case.eval_rules) - - implicit_config: list[EvalRuleConfig] = [] - if case.expectations.response_time_max_ms: - implicit_config.append( - EvalRuleConfig( - type="response_time", - params={ - "max_ms": case.expectations.response_time_max_ms, - }, - ) - ) - if case.expectations.keywords_include or case.expectations.keywords_exclude: - implicit_config.append( - EvalRuleConfig( - type="keyword_match", - params={ - "keywords": case.expectations.keywords_include, - "exclude_keywords": case.expectations.keywords_exclude, - }, - ) - ) + implicit_config = derive_implicit_rules(case.expectations) all_replied = bool(dialog) and all(t.reply is not None for t in dialog) @@ -415,11 +423,7 @@ class EvalEngine: all_rules = [(cfg, False) for cfg in rules_config] + [(cfg, True) for cfg in implicit_config] for rule_config, is_implicit in all_rules: - purpose = { - "llm_score": ModelPurpose.JUDGE, - "semantic_similarity": ModelPurpose.EMBEDDING, - "safety": ModelPurpose.MODERATION, - }.get(rule_config.type) + purpose = RULE_PURPOSE.get(rule_config.type) try: model_config = self._resolve_model(purpose) if purpose else None rule = get_rule( diff --git a/backend/agenteval/evaluation/implicit_rules.py b/backend/agenteval/evaluation/implicit_rules.py new file mode 100644 index 0000000..fdc8172 --- /dev/null +++ b/backend/agenteval/evaluation/implicit_rules.py @@ -0,0 +1,30 @@ +"""Translate a case's期望 (Expectation) into implicit评估规则. + +期望描述"想要什么"，评估规则是"怎么判定"——期望派生的隐式规则与显式规则 +叠加生效（CONTEXT.md）。此翻译是纯逻辑，独立于规则执行与持久化，便于单测。 +""" + +from agenteval.models import EvalRuleConfig, Expectation + + +def derive_implicit_rules(expectation: Expectation) -> list[EvalRuleConfig]: + """Build the implicit rule configs a case's expectation implies.""" + rules: list[EvalRuleConfig] = [] + if expectation.response_time_max_ms: + rules.append( + EvalRuleConfig( + type="response_time", + params={"max_ms": expectation.response_time_max_ms}, + ) + ) + if expectation.keywords_include or expectation.keywords_exclude: + rules.append( + EvalRuleConfig( + type="keyword_match", + params={ + "keywords": expectation.keywords_include, + "exclude_keywords": expectation.keywords_exclude, + }, + ) + ) + return rules diff --git a/tests/unit/test_implicit_rules.py b/tests/unit/test_implicit_rules.py new file mode 100644 index 0000000..5224811 --- /dev/null +++ b/tests/unit/test_implicit_rules.py @@ -0,0 +1,45 @@ +"""Unit tests for derive_implicit_rules — 期望→隐式规则的纯翻译.""" + +from agenteval.evaluation.implicit_rules import derive_implicit_rules +from agenteval.models import Expectation + + +def test_empty_expectation_yields_no_rules(): + assert derive_implicit_rules(Expectation()) == [] + + +def test_intent_and_coherence_alone_yield_no_rules(): + # intent / coherence_min_score 不派生隐式规则（无对应规则类型） + exp = Expectation(intent="问诊分流", coherence_min_score=0.8) + assert derive_implicit_rules(exp) == [] + + +def test_response_time_yields_response_time_rule(): + rules = derive_implicit_rules(Expectation(response_time_max_ms=2000)) + assert len(rules) == 1 + assert rules[0].type == "response_time" + assert rules[0].params == {"max_ms": 2000} + + +def test_keywords_include_yields_keyword_rule(): + rules = derive_implicit_rules(Expectation(keywords_include=["挂号", "门诊"])) + assert len(rules) == 1 + assert rules[0].type == "keyword_match" + assert rules[0].params == {"keywords": ["挂号", "门诊"], "exclude_keywords": []} + + +def test_keywords_exclude_alone_yields_keyword_rule(): + rules = derive_implicit_rules(Expectation(keywords_exclude=["投诉"])) + assert len(rules) == 1 + assert rules[0].type == "keyword_match" + assert rules[0].params == {"keywords": [], "exclude_keywords": ["投诉"]} + + +def test_both_expectations_yield_two_rules_in_order(): + exp = Expectation( + response_time_max_ms=1500, + keywords_include=["预约"], + keywords_exclude=["取消"], + ) + rules = derive_implicit_rules(exp) + assert [r.type for r in rules] == ["response_time", "keyword_match"]