From 12481cd1b84afdb676618bd9b8bd21da7331552e Mon Sep 17 00:00:00 2001 From: sinohqb Date: Fri, 17 Jul 2026 10:52:32 +0800 Subject: [PATCH] =?UTF-8?q?v0.3-s1:=20=E8=A7=84=E5=88=99=E5=B1=82=E5=BC=82?= =?UTF-8?q?=E6=AD=A5=E5=8C=96=20+=20=E5=B7=A5=E5=85=B7=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E5=8E=BB=E9=87=8D=20+=20HTTP=20=E9=80=9A=E9=81=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 核心变更 ### 规则层全面异步化（DEBT-1） - EvalRule.evaluate() 签名改为 async def，全量同步改造（无兼容层） - LlmScoreRule._call_llm: requests.post → httpx.AsyncClient，彻底消除事件循环阻塞 - engine._save_rule_results: rule.evaluate() → await rule.evaluate() ### 工具函数去重（DEBT-2） - 新建 agenteval/utils/llm.py，统一三个函数： - extract_reply_text (原 5 处重复) - extract_content_from_llm_response (原 2 处重复) - parse_json_from_llm_text (统一 LLM 输出 JSON 解析) - engine.py / llm_score.py / runs.py / report.py 全部切换到 utils.llm ### HTTP 通用通道（S1-3） - 新建 channels/http.py (HttpChannel) - 配置化 send_url / reply_url 模板 ({message}, {msg_id} 占位) - dot-path 提取 msg_id 和 reply_text - 可选 reply_ready_path 就绪标志 - 长连接 AsyncClient 复用 - ChannelFactory 注册 ChannelType.HTTP → HttpChannel ### 测试 - 新增 tests/unit/test_http_channel_and_rules.py (19 个测试) - _get_path / health_check / send / poll_reply / 超时 / 就绪标志 / async 规则评估 - 测试总数：24 → 43，全部通过 Co-Authored-By: Claude --- backend/agenteval/channels/factory.py | 2 + backend/agenteval/channels/http.py | 132 ++++++++ backend/agenteval/config/settings.py | 1 - backend/agenteval/evaluation/engine.py | 308 +++++++++++------- backend/agenteval/evaluation/report.py | 12 +- backend/agenteval/evaluation/rules/base.py | 4 +- backend/agenteval/evaluation/rules/keyword.py | 21 +- .../agenteval/evaluation/rules/llm_score.py | 75 +---- .../evaluation/rules/response_time.py | 2 +- backend/agenteval/storage/db.py | 1 - backend/agenteval/storage/file_repository.py | 4 +- backend/agenteval/utils/__init__.py | 0 backend/agenteval/utils/llm.py | 72 ++++ backend/agenteval/web/app.py | 1 + backend/agenteval/web/deps.py | 1 - backend/agenteval/web/routers/files.py | 4 +- backend/agenteval/web/routers/runs.py | 24 +- backend/cli/run.py | 4 +- docs/plan-v0.3.md | 195 +++++++++++ tests/unit/test_http_channel_and_rules.py | 213 ++++++++++++ 20 files changed, 833 insertions(+), 243 deletions(-) create mode 100644 backend/agenteval/channels/http.py create mode 100644 backend/agenteval/utils/__init__.py create mode 100644 backend/agenteval/utils/llm.py create mode 100644 docs/plan-v0.3.md create mode 100644 tests/unit/test_http_channel_and_rules.py diff --git a/backend/agenteval/channels/factory.py b/backend/agenteval/channels/factory.py index 95723ad..efb6dce 100644 --- a/backend/agenteval/channels/factory.py +++ b/backend/agenteval/channels/factory.py @@ -1,6 +1,7 @@ """Factory for creating message channels from target configuration.""" from agenteval.channels.base import EvalChannel +from agenteval.channels.http import HttpChannel from agenteval.channels.tutu import TutuApiChannel from agenteval.models import ChannelType, EvalTarget @@ -10,6 +11,7 @@ class ChannelFactory: _mapping: dict[ChannelType, type[EvalChannel]] = { ChannelType.TUTU_API: TutuApiChannel, + ChannelType.HTTP: HttpChannel, } @classmethod diff --git a/backend/agenteval/channels/http.py b/backend/agenteval/channels/http.py new file mode 100644 index 0000000..64916d5 --- /dev/null +++ b/backend/agenteval/channels/http.py @@ -0,0 +1,132 @@ +"""Generic HTTP evaluation channel. + +Supports any REST API that follows a configurable request/response template. +Configuration keys (all under channel_config): + +Required: + send_url URL to POST the message to (supports {message} template var) + reply_url URL to GET the reply from (supports {msg_id} template var) + +Optional: + health_url URL for health check GET (defaults to send_url) + headers dict of extra request headers (e.g. Authorization) + send_body JSON body template for POST; {message} is substituted + Default: {"text": "{message}"} + msg_id_path dot-separated path to extract msg_id from send response + Default: "id" + reply_path dot-separated path to extract reply text from reply response + Default: "text" + reply_ready_path dot-separated path to a boolean indicating reply is ready + Default: (treat any 200 as ready) + poll_interval seconds between polls (default 1.0) + timeout seconds before poll gives up (default 30.0) +""" + +import asyncio +import json +import uuid +from typing import Any, Optional + +import httpx + +from agenteval.channels.base import ChannelHealth, EvalChannel, Reply, SendResult + + +def _get_path(data: Any, path: str) -> Any: + """Traverse a dot-separated key path into a nested dict/list.""" + parts = path.split(".") + current = data + for part in parts: + if current is None: + return None + if isinstance(current, dict): + current = current.get(part) + elif isinstance(current, list) and part.isdigit(): + idx = int(part) + current = current[idx] if idx < len(current) else None + else: + return None + return current + + +class HttpChannel(EvalChannel): + """Generic HTTP channel: POST to send, GET/POST to poll for reply.""" + + def __init__(self, config: dict[str, Any]): + self.send_url: str = config["send_url"] + self.reply_url: str = config.get("reply_url", "") + self.health_url: str = config.get("health_url", self.send_url) + self.headers: dict[str, str] = config.get("headers", {}) + self.send_body_template: str = config.get("send_body", '{"text": "{message}"}') + self.msg_id_path: str = config.get("msg_id_path", "id") + self.reply_path: str = config.get("reply_path", "text") + self.reply_ready_path: Optional[str] = config.get("reply_ready_path") + self._poll_interval: float = float(config.get("poll_interval", 1.0)) + + self._client = httpx.AsyncClient(headers=self.headers, timeout=30) + + async def close(self) -> None: + await self._client.aclose() + + async def health_check(self) -> ChannelHealth: + try: + resp = await self._client.get(self.health_url) + resp.raise_for_status() + return ChannelHealth(ok=True, message=f"HTTP {resp.status_code}") + except Exception as exc: + return ChannelHealth(ok=False, message=str(exc)) + + async def send(self, content: str, **kwargs: Any) -> SendResult: + body_str = self.send_body_template.replace("{message}", content) + try: + body = json.loads(body_str) + except json.JSONDecodeError: + body = {"text": content} + + url = self.send_url.replace("{message}", content) + try: + resp = await self._client.post(url, json=body) + resp.raise_for_status() + data = resp.json() + msg_id = str(_get_path(data, self.msg_id_path) or uuid.uuid4()) + return SendResult(ok=True, question_msg_id=msg_id, raw_response=data) + except Exception as exc: + return SendResult(ok=False, error=str(exc)) + + async def poll_reply( + self, + question_msg_id: str, + timeout: float = 30.0, + poll_interval: float = 1.0, + ) -> Optional[Reply]: + interval = poll_interval or self._poll_interval + deadline = asyncio.get_event_loop().time() + timeout + url = self.reply_url.replace("{msg_id}", question_msg_id) + + while asyncio.get_event_loop().time() < deadline: + try: + resp = await self._client.get(url) + resp.raise_for_status() + data = resp.json() + + # Check if reply is ready (optional readiness flag) + if self.reply_ready_path: + ready = _get_path(data, self.reply_ready_path) + if not ready: + await asyncio.sleep(interval) + continue + + reply_text = _get_path(data, self.reply_path) + if reply_text is not None: + raw = {"text": str(reply_text), "_raw": data} + return Reply( + question_msg_id=question_msg_id, + content=str(reply_text), + raw_message=raw, + ) + except Exception: + pass + + await asyncio.sleep(interval) + + return None diff --git a/backend/agenteval/config/settings.py b/backend/agenteval/config/settings.py index ef919ef..34103fa 100644 --- a/backend/agenteval/config/settings.py +++ b/backend/agenteval/config/settings.py @@ -16,7 +16,6 @@ from typing import Optional from pydantic import Field, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict - ROOT_DIR = Path(__file__).resolve().parent.parent.parent.parent ENV_FILE = ROOT_DIR / ".env" diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index ebb4799..7d9084f 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -6,10 +6,9 @@ via an ``asyncio.Event`` cancel token. """ import asyncio -import json import uuid -from dataclasses import dataclass, field -from typing import Any, Awaitable, Callable, Optional +from dataclasses import dataclass +from typing import Any, Callable, Optional import httpx @@ -19,7 +18,7 @@ from agenteval.evaluation.rules import get_rule from agenteval.models import Case, CaseType, EvalResult, EvalRun, EvalTarget, RunStatus, Scenario, Turn from agenteval.storage.db import get_session, utc_now from agenteval.storage.repository import ResultRepository, RunRepository - +from agenteval.utils.llm import extract_content_from_llm_response, extract_reply_text, parse_json_from_llm_text # Progress callbacks may be sync or async; the engine awaits the result if # it is a coroutine, otherwise treats it as a plain function. @@ -45,48 +44,6 @@ def _build_send_message(content: str) -> dict[str, Any]: } -def _extract_content_from_api_response(data: dict) -> str: - """Extract text content from an LLM API response. - - Handles both OpenAI format (choices[0].message.content as string) - and content-block-array format used by Anthropic-compatible APIs - (choices[0].message.content as list of {type, text/text} blocks). - """ - try: - content = data["choices"][0]["message"]["content"] - except (KeyError, IndexError, TypeError): - return "" - if isinstance(content, str): - return content - if isinstance(content, list): - parts = [] - for block in content: - if not isinstance(block, dict): - continue - # Anthropic format: {"type": "text", "text": "..."} - # Some APIs may use "content" key instead of "text" - if block.get("type") == "text": - parts.append(block.get("text") or block.get("content") or "") - else: - # Non-text blocks (tool_use, image, etc.) — skip - pass - return "\n".join(parts) - return str(content) - - -def _extract_reply_text(reply: Any) -> str: - if reply is None: - return "" - if isinstance(reply, str): - return reply - if isinstance(reply, dict): - body = reply.get("msgBody") or reply.get("content", "") - if isinstance(body, dict): - return body.get("content", "") - return str(body) - return str(reply) - - class EvalEngine: """Execute evaluation scenarios against targets. @@ -151,22 +108,37 @@ class EvalEngine: for idx, case in enumerate(self.scenario.cases, start=1): self._check_cancel() - await self._emit(progress_callback, "case_start", { - "index": idx, "total": total_cases, "case_id": case.id, - }) + await self._emit( + progress_callback, + "case_start", + { + "index": idx, + "total": total_cases, + "case_id": case.id, + }, + ) async with self._case_semaphore: case_passed, rule_pass, rule_total = await self._run_case( - run, case, progress_callback, + run, + case, + progress_callback, ) if case_passed: passed_cases += 1 else: failed_cases += 1 - await self._emit(progress_callback, "case_end", { - "index": idx, "total": total_cases, "case_id": case.id, - "passed": case_passed, - "rule_pass_count": rule_pass, "rule_total": rule_total, - }) + await self._emit( + progress_callback, + "case_end", + { + "index": idx, + "total": total_cases, + "case_id": case.id, + "passed": case_passed, + "rule_pass_count": rule_pass, + "rule_total": rule_total, + }, + ) results = self.run_repo.get_results(run.id) total_rules = len(results) @@ -183,27 +155,42 @@ class EvalEngine: run.status = RunStatus.COMPLETED run.completed_at = utc_now() run.summary = summary - await self._emit(progress_callback, "run_completed", { - "status": "completed", "summary": summary, - }) + await self._emit( + progress_callback, + "run_completed", + { + "status": "completed", + "summary": summary, + }, + ) except CancelledError: run.status = RunStatus.FAILED run.completed_at = utc_now() run.summary = { "error": {"code": "cancelled_by_user", "message": "评测已手动停止"}, } - await self._emit(progress_callback, "run_completed", { - "status": "failed", "reason": "cancelled", - "error": {"code": "cancelled_by_user", "message": "评测已手动停止"}, - }) + await self._emit( + progress_callback, + "run_completed", + { + "status": "failed", + "reason": "cancelled", + "error": {"code": "cancelled_by_user", "message": "评测已手动停止"}, + }, + ) except Exception as exc: run.status = RunStatus.FAILED run.completed_at = utc_now() run.summary = {"error": str(exc)} await self._emit(progress_callback, "error", {"error": str(exc)}) - await self._emit(progress_callback, "run_completed", { - "status": "failed", "error": str(exc), - }) + await self._emit( + progress_callback, + "run_completed", + { + "status": "failed", + "error": str(exc), + }, + ) raise finally: run = self.run_repo.update(run) or run @@ -235,9 +222,14 @@ class EvalEngine: if case.type == CaseType.DYNAMIC: generated = await self._generate_messages(case, progress_callback) if not generated: - await self._emit(progress_callback, "error", { - "error": "LLM 未能生成测试消息", "case_id": case.id, - }) + await self._emit( + progress_callback, + "error", + { + "error": "LLM 未能生成测试消息", + "case_id": case.id, + }, + ) return False, 0, 0 case = case.model_copy(update={"messages": generated}) @@ -245,24 +237,39 @@ class EvalEngine: for round_index, message in enumerate(case.messages, start=1): self._check_cancel() - await self._emit(progress_callback, "turn_start", { - "run_id": run.id, "case_id": case.id, - "round": round_index, "message": message, - }) + await self._emit( + progress_callback, + "turn_start", + { + "run_id": run.id, + "case_id": case.id, + "round": round_index, + "message": message, + }, + ) sent_at = utc_now() send_result = await self.channel.send(message) if not send_result.ok: turn = Turn( id=str(uuid.uuid4()), - run_id=run.id, case_id=case.id, round_index=round_index, - sent_message=_build_send_message(message), sent_at=sent_at, + run_id=run.id, + case_id=case.id, + round_index=round_index, + sent_message=_build_send_message(message), + sent_at=sent_at, ) self.result_repo.save_turn(turn) await self._save_rule_results(run, case, turn, [], progress_callback) - await self._emit(progress_callback, "turn_error", { - "case_id": case.id, "round": round_index, "error": send_result.error, - }) + await self._emit( + progress_callback, + "turn_error", + { + "case_id": case.id, + "round": round_index, + "error": send_result.error, + }, + ) return False, 0, 0 try: @@ -274,16 +281,24 @@ class EvalEngine: received_at = utc_now() turn = Turn( id=str(uuid.uuid4()), - run_id=run.id, case_id=case.id, round_index=round_index, - sent_message=_build_send_message(message), sent_at=sent_at, + run_id=run.id, + case_id=case.id, + round_index=round_index, + sent_message=_build_send_message(message), + sent_at=sent_at, question_msg_id=send_result.question_msg_id, received_at=received_at, ) self.result_repo.save_turn(turn) - await self._emit(progress_callback, "turn_error", { - "case_id": case.id, "round": round_index, - "error": f"poll_reply 异常: {poll_exc}", - }) + await self._emit( + progress_callback, + "turn_error", + { + "case_id": case.id, + "round": round_index, + "error": f"poll_reply 异常: {poll_exc}", + }, + ) return False, 0, 0 received_at = utc_now() @@ -293,20 +308,30 @@ class EvalEngine: turn = Turn( id=str(uuid.uuid4()), - run_id=run.id, case_id=case.id, round_index=round_index, - sent_message=_build_send_message(message), sent_at=sent_at, + run_id=run.id, + case_id=case.id, + round_index=round_index, + sent_message=_build_send_message(message), + sent_at=sent_at, question_msg_id=send_result.question_msg_id, reply=reply.raw_message if reply else None, - received_at=received_at, latency_ms=latency_ms, + received_at=received_at, + latency_ms=latency_ms, ) self.result_repo.save_turn(turn) dialog.append(turn) - await self._emit(progress_callback, "turn_end", { - "run_id": run.id, "case_id": case.id, "round": round_index, - "latency_ms": latency_ms, - "reply_text": _extract_reply_text(reply.raw_message if reply else None), - }) + await self._emit( + progress_callback, + "turn_end", + { + "run_id": run.id, + "case_id": case.id, + "round": round_index, + "latency_ms": latency_ms, + "reply_text": extract_reply_text(reply.raw_message if reply else None), + }, + ) if not dialog: return False, 0, 0 @@ -329,9 +354,14 @@ class EvalEngine: # If no explicit rules, derive implicit rules from expectations. if not rules_config: if case.expectations.response_time_max_ms: - rules_config.append(EvalRuleConfig(type="response_time", params={ - "max_ms": case.expectations.response_time_max_ms, - })) + rules_config.append( + EvalRuleConfig( + type="response_time", + params={ + "max_ms": case.expectations.response_time_max_ms, + }, + ) + ) if case.expectations.keywords_include or case.expectations.keywords_exclude: rules_config.append( EvalRuleConfig( @@ -348,12 +378,16 @@ class EvalEngine: total_count = 0 for rule_config in rules_config: rule = get_rule(rule_config.type, rule_config.params) - result = rule.evaluate(case, dialog) + result = await rule.evaluate(case, dialog) eval_result = EvalResult( id=str(uuid.uuid4()), - run_id=run.id, case_id=case.id, turn_id=turn.id or "", + run_id=run.id, + case_id=case.id, + turn_id=turn.id or "", rule_type=rule_config.type, - passed=result.passed, score=result.score, reason=result.reason, + passed=result.passed, + score=result.score, + reason=result.reason, ) self.result_repo.save_result(eval_result) total_count += 1 @@ -361,23 +395,36 @@ class EvalEngine: passed_count += 1 else: all_passed = False - await self._emit(progress_callback, "rule_result", { - "run_id": run.id, "case_id": case.id, - "rule_type": rule_config.type, - "passed": result.passed, "score": result.score, "reason": result.reason, - }) + await self._emit( + progress_callback, + "rule_result", + { + "run_id": run.id, + "case_id": case.id, + "rule_type": rule_config.type, + "passed": result.passed, + "score": result.score, + "reason": result.reason, + }, + ) return all_passed, passed_count, total_count async def _generate_messages( - self, case: Case, progress_callback: Optional[ProgressCallback], + self, + case: Case, + progress_callback: Optional[ProgressCallback], ) -> list[str]: """Use LLM to generate test messages for dynamic cases.""" llm_config = self.scenario.llm_config if not llm_config: - await self._emit(progress_callback, "error", { - "error": "动态用例需要配置 llm_config", - }) + await self._emit( + progress_callback, + "error", + { + "error": "动态用例需要配置 llm_config", + }, + ) return [] api_url = llm_config.get("api_url") @@ -415,25 +462,28 @@ class EvalEngine: async with httpx.AsyncClient(timeout=self.timeout_config.llm_generate) as client: resp = await client.post(api_url, headers=headers, json=payload) resp.raise_for_status() - data = resp.json() - content = _extract_content_from_api_response(data) + content = extract_content_from_llm_response(resp.json()) if not content: - await self._emit(progress_callback, "error", { - "error": "LLM 返回内容为空或无法解析", - }) + await self._emit( + progress_callback, + "error", + { + "error": "LLM 返回内容为空或无法解析", + }, + ) return [] try: - parsed = json.loads(content) - except json.JSONDecodeError: - start = content.find("[") - end = content.rfind("]") - if start == -1 or end == -1: - await self._emit(progress_callback, "error", { - "error": f"LLM 返回无法解析为数组: {content[:200]}", - }) - return [] - parsed = json.loads(content[start:end + 1]) + parsed = parse_json_from_llm_text(content) + except (ValueError, Exception) as parse_exc: + await self._emit( + progress_callback, + "error", + { + "error": f"LLM 返回无法解析为数组: {parse_exc}", + }, + ) + return [] if not isinstance(parsed, list): await self._emit(progress_callback, "error", {"error": "LLM 返回的不是数组"}) @@ -444,9 +494,14 @@ class EvalEngine: await self._emit(progress_callback, "error", {"error": "LLM 返回的消息为空"}) return [] - await self._emit(progress_callback, "messages_generated", { - "case_id": case.id, "messages": messages, - }) + await self._emit( + progress_callback, + "messages_generated", + { + "case_id": case.id, + "messages": messages, + }, + ) return messages except Exception as exc: @@ -460,7 +515,10 @@ class EvalEngine: raise CancelledError("run cancelled") async def _emit( - self, callback: Optional[ProgressCallback], event: str, data: dict[str, Any], + self, + callback: Optional[ProgressCallback], + event: str, + data: dict[str, Any], ) -> None: if not callback: return diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index 10e5d19..7952116 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -9,6 +9,7 @@ from jinja2 import Template from agenteval.storage.db import DATA_DIR from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository +from agenteval.utils.llm import extract_reply_text HTML_TEMPLATE = """ @@ -88,16 +89,7 @@ HTML_TEMPLATE = """ def _extract_text(data: Any) -> str: - if data is None: - return "" - if isinstance(data, str): - return data - if isinstance(data, dict): - body = data.get("msgBody") or data.get("content", "") - if isinstance(body, dict): - return body.get("content", "") - return str(body) - return str(data) + return extract_reply_text(data) def generate_report(run_id: str, session=None) -> dict[str, Any]: diff --git a/backend/agenteval/evaluation/rules/base.py b/backend/agenteval/evaluation/rules/base.py index 2e4144b..4241307 100644 --- a/backend/agenteval/evaluation/rules/base.py +++ b/backend/agenteval/evaluation/rules/base.py @@ -17,7 +17,7 @@ class RuleResult: class EvalRule(ABC): - """Abstract evaluation rule.""" + """Abstract evaluation rule. All implementations must be async.""" name: str = "" @@ -25,7 +25,7 @@ class EvalRule(ABC): self.params = params @abstractmethod - def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult: + async def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult: """Evaluate the dialog against the case expectation.""" ... diff --git a/backend/agenteval/evaluation/rules/keyword.py b/backend/agenteval/evaluation/rules/keyword.py index a573d19..e696fec 100644 --- a/backend/agenteval/evaluation/rules/keyword.py +++ b/backend/agenteval/evaluation/rules/keyword.py @@ -2,21 +2,7 @@ from agenteval.evaluation.rules.base import EvalRule, RuleResult, register_rule from agenteval.models import Case, Turn - - -def _extract_text(reply) -> str: - """Extract plain text from a reply object for matching.""" - if reply is None: - return "" - if isinstance(reply, str): - return reply - if isinstance(reply, dict): - # Tutu-api message structure: msgBody.content - body = reply.get("msgBody") or reply.get("content", "") - if isinstance(body, dict): - return body.get("content", "") - return str(body) - return str(reply) +from agenteval.utils.llm import extract_reply_text @register_rule @@ -25,13 +11,12 @@ class KeywordMatchRule(EvalRule): name = "keyword_match" - def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult: + async def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult: if not dialog: return RuleResult(passed=False, reason="无回复记录") last_turn = dialog[-1] - reply = last_turn.reply - text = _extract_text(reply).lower() + text = extract_reply_text(last_turn.reply).lower() params = self.params include = [k.lower() for k in params.get("keywords", [])] diff --git a/backend/agenteval/evaluation/rules/llm_score.py b/backend/agenteval/evaluation/rules/llm_score.py index a067184..9193352 100644 --- a/backend/agenteval/evaluation/rules/llm_score.py +++ b/backend/agenteval/evaluation/rules/llm_score.py @@ -1,49 +1,12 @@ """LLM-based scoring evaluation rule.""" import json -from typing import Any -import requests +import httpx from agenteval.evaluation.rules.base import EvalRule, RuleResult, register_rule from agenteval.models import Case, Turn - - -def _extract_text(reply: Any) -> str: - if reply is None: - return "" - if isinstance(reply, str): - return reply - if isinstance(reply, dict): - body = reply.get("msgBody") or reply.get("content", "") - if isinstance(body, dict): - return body.get("content", "") - return str(body) - return str(reply) - - -def _extract_content_from_api_response(data: dict) -> str: - """Extract text content from an LLM API response. - - Handles both OpenAI format (choices[0].message.content as string) - and content-block-array format used by Anthropic-compatible APIs - (choices[0].message.content as list of {type, text/text} blocks). - """ - try: - content = data["choices"][0]["message"]["content"] - except (KeyError, IndexError, TypeError): - return "" - if isinstance(content, str): - return content - if isinstance(content, list): - parts = [] - for block in content: - if not isinstance(block, dict): - continue - if block.get("type") == "text": - parts.append(block.get("text") or block.get("content") or "") - return "\n".join(parts) - return str(content) +from agenteval.utils.llm import extract_content_from_llm_response, extract_reply_text, parse_json_from_llm_text @register_rule @@ -52,23 +15,23 @@ class LlmScoreRule(EvalRule): name = "llm_score" - def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult: + async def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult: if not dialog: return RuleResult(passed=False, reason="无回复记录") last_turn = dialog[-1] - reply_text = _extract_text(last_turn.reply) + reply_text = extract_reply_text(last_turn.reply) + question_text = "" if len(dialog) >= 2: - question_text = _extract_text(dialog[-2].reply) or "" + question_text = extract_reply_text(dialog[-2].reply) or "" if not question_text and last_turn.sent_message: body = last_turn.sent_message.get("msgBody", "") if isinstance(body, dict): question_text = body.get("content", "") else: try: - parsed = json.loads(body) - question_text = parsed.get("content", "") + question_text = json.loads(body).get("content", "") except Exception: question_text = str(body) @@ -81,7 +44,7 @@ class LlmScoreRule(EvalRule): if not api_url: return RuleResult(passed=False, reason="LLM 评分规则未配置 api_url") - score, reason = self._call_llm(api_url, api_key, model, question_text, reply_text, criteria) + score, reason = await self._call_llm(api_url, api_key, model, question_text, reply_text, criteria) if score is None: return RuleResult(passed=False, reason=f"LLM 评分失败: {reason}") @@ -92,7 +55,7 @@ class LlmScoreRule(EvalRule): reason=f"LLM 评分 {score}/10，{'通过' if passed else '未通过'} (阈值 {min_score})", ) - def _call_llm( + async def _call_llm( self, api_url: str, api_key: str | None, @@ -105,7 +68,7 @@ class LlmScoreRule(EvalRule): system_prompt = ( "你是一位严格的智能客服质量评估专家。请根据用户问题和智能体回复，" f"按照以下标准打分（0-10分，10分最高）：{criteria}\n" - "只输出一个 JSON 对象：{\"score\": number, \"reason\": \"简短说明\"}" + '只输出一个 JSON 对象：{"score": number, "reason": "简短说明"}' ) user_prompt = f"用户问题：{question}\n智能体回复：{reply}" @@ -123,24 +86,14 @@ class LlmScoreRule(EvalRule): } try: - resp = requests.post(api_url, headers=headers, json=payload, timeout=60) + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.post(api_url, headers=headers, json=payload) resp.raise_for_status() - data = resp.json() - content = _extract_content_from_api_response(data) + content = extract_content_from_llm_response(resp.json()) if not content: return None, "LLM 返回内容为空" - # Try to parse JSON from the content - try: - parsed = json.loads(content) - except json.JSONDecodeError: - # Fallback: extract JSON substring - start = content.find("{") - end = content.rfind("}") - if start == -1 or end == -1: - return None, "LLM 返回格式无法解析" - parsed = json.loads(content[start : end + 1]) - + parsed = parse_json_from_llm_text(content) score = float(parsed["score"]) reason = parsed.get("reason", "") return max(0.0, min(10.0, score)), reason diff --git a/backend/agenteval/evaluation/rules/response_time.py b/backend/agenteval/evaluation/rules/response_time.py index 0404260..acbbea8 100644 --- a/backend/agenteval/evaluation/rules/response_time.py +++ b/backend/agenteval/evaluation/rules/response_time.py @@ -10,7 +10,7 @@ class ResponseTimeRule(EvalRule): name = "response_time" - def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult: + async def evaluate(self, case: Case, dialog: list[Turn]) -> RuleResult: if not dialog: return RuleResult(passed=False, reason="无回复记录") diff --git a/backend/agenteval/storage/db.py b/backend/agenteval/storage/db.py index bc8deda..98049a7 100644 --- a/backend/agenteval/storage/db.py +++ b/backend/agenteval/storage/db.py @@ -3,7 +3,6 @@ import json import uuid from datetime import datetime, timezone -import os from pathlib import Path from typing import Any, Optional diff --git a/backend/agenteval/storage/file_repository.py b/backend/agenteval/storage/file_repository.py index e19dc98..8fea3e4 100644 --- a/backend/agenteval/storage/file_repository.py +++ b/backend/agenteval/storage/file_repository.py @@ -6,9 +6,9 @@ from typing import Optional from sqlmodel import Session, select from agenteval.storage.db import ( + FILES_DIR, FileCategoryDB, FileRecordDB, - FILES_DIR, get_session, new_uuid, utc_now, @@ -183,4 +183,4 @@ class FileRecordRepository: return file_path = FILES_DIR / record.storage_name if file_path.exists(): - os.remove(file_path) \ No newline at end of file + os.remove(file_path) diff --git a/backend/agenteval/utils/__init__.py b/backend/agenteval/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/agenteval/utils/llm.py b/backend/agenteval/utils/llm.py new file mode 100644 index 0000000..95acbbf --- /dev/null +++ b/backend/agenteval/utils/llm.py @@ -0,0 +1,72 @@ +"""Shared utilities for LLM API interaction and message extraction. + +Consolidates the duplicated _extract_text / _extract_reply_text pattern +(previously repeated in 5 places) and _extract_content_from_api_response +(previously duplicated in 2 places). +""" + +from typing import Any + + +def extract_reply_text(reply: Any) -> str: + """Extract plain text from a tutu-api reply object. + + Handles: None | str | dict with msgBody.content or content key. + """ + if reply is None: + return "" + if isinstance(reply, str): + return reply + if isinstance(reply, dict): + body = reply.get("msgBody") or reply.get("content", "") + if isinstance(body, dict): + return body.get("content", "") + return str(body) + return str(reply) + + +def extract_content_from_llm_response(data: dict) -> str: + """Extract text content from an LLM API response dict. + + Handles: + - OpenAI format: choices[0].message.content as a plain string + - Anthropic-compatible format: choices[0].message.content as a list of + content blocks {type: "text", text: "..."} (non-text blocks are skipped) + """ + try: + content = data["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError): + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") == "text": + parts.append(block.get("text") or block.get("content") or "") + return "\n".join(parts) + return str(content) + + +def parse_json_from_llm_text(content: str) -> Any: + """Parse JSON from LLM output, falling back to bracket-delimited substring. + + Returns parsed JSON object, or raises json.JSONDecodeError if unparseable. + """ + import json + + try: + return json.loads(content) + except json.JSONDecodeError: + pass + + # Fallback: find the outermost JSON object or array + for open_char, close_char in [("{", "}"), ("[", "]")]: + start = content.find(open_char) + end = content.rfind(close_char) + if start != -1 and end != -1 and end > start: + return json.loads(content[start : end + 1]) + + raise ValueError(f"No JSON found in LLM output: {content[:200]}") diff --git a/backend/agenteval/web/app.py b/backend/agenteval/web/app.py index 0c5c584..3ea5afd 100644 --- a/backend/agenteval/web/app.py +++ b/backend/agenteval/web/app.py @@ -70,6 +70,7 @@ _default_dist = Path(__file__).resolve().parent.parent.parent.parent / "frontend WEB_DIST = Path(settings.frontend_dist_path) if settings.frontend_dist_path else _default_dist if WEB_DIST.exists(): from fastapi.staticfiles import StaticFiles + app.mount("/assets", StaticFiles(directory=WEB_DIST / "assets"), name="assets") diff --git a/backend/agenteval/web/deps.py b/backend/agenteval/web/deps.py index dfcbece..84170dd 100644 --- a/backend/agenteval/web/deps.py +++ b/backend/agenteval/web/deps.py @@ -7,7 +7,6 @@ longer re-declare these helpers themselves. from typing import Optional from fastapi import Depends, Header, HTTPException, status -from sqlmodel import Session from agenteval.config import get_settings from agenteval.storage.db import get_session diff --git a/backend/agenteval/web/routers/files.py b/backend/agenteval/web/routers/files.py index b795571..82c3cb6 100644 --- a/backend/agenteval/web/routers/files.py +++ b/backend/agenteval/web/routers/files.py @@ -10,7 +10,7 @@ from pydantic import BaseModel from sqlmodel import Session from agenteval.config import get_settings -from agenteval.storage.db import FILES_DIR, FileRecordDB +from agenteval.storage.db import FILES_DIR from agenteval.storage.file_repository import FileCategoryRepository, FileRecordRepository from agenteval.web.deps import get_db @@ -200,4 +200,4 @@ def download_file(file_id: str, session: Session = Depends(get_db)): def delete_file(file_id: str, session: Session = Depends(get_db)) -> dict: if not FileRecordRepository(session).delete(file_id): raise HTTPException(status_code=404, detail="文件不存在") - return {"ok": True} \ No newline at end of file + return {"ok": True} diff --git a/backend/agenteval/web/routers/runs.py b/backend/agenteval/web/routers/runs.py index cba949a..9cd585e 100644 --- a/backend/agenteval/web/routers/runs.py +++ b/backend/agenteval/web/routers/runs.py @@ -11,6 +11,7 @@ from agenteval.evaluation.engine import EvalEngine from agenteval.models import EvalRun, RunStatus from agenteval.storage.db import get_session from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository +from agenteval.utils.llm import extract_reply_text from agenteval.web.deps import get_db from agenteval.web.websocket import ws_manager @@ -42,7 +43,9 @@ async def _run_evaluation(run_id: str, target_id: str, scenario_id: str) -> None return engine = EvalEngine( - target=target, scenario=scenario, session=session, + target=target, + scenario=scenario, + session=session, cancel_token=cancel_token, ) await engine.run( @@ -135,7 +138,7 @@ async def get_run_logs(run_id: str, session: Session = Depends(get_db)) -> dict: "round_index": t.round_index, "latency_ms": t.latency_ms, "sent_text": t.get_sent_message().get("msgBody", {}).get("content", ""), - "reply_text": _extract_reply_text(t.get_reply()), + "reply_text": extract_reply_text(t.get_reply()), "sent_at": t.sent_at.isoformat() if t.sent_at else None, "received_at": t.received_at.isoformat() if t.received_at else None, } @@ -169,22 +172,7 @@ async def get_run_logs(run_id: str, session: Session = Depends(get_db)) -> dict: "response_time_max_ms": case.expectations.response_time_max_ms, "coherence_min_score": case.expectations.coherence_min_score, }, - "eval_rules": [ - {"type": r.type, "params": dict(r.params)} for r in case.eval_rules - ], + "eval_rules": [{"type": r.type, "params": dict(r.params)} for r in case.eval_rules], } return {"turns": turns_data, "results": results_data, "scenario_snapshot": scenario_snapshot} - - -def _extract_reply_text(reply) -> str: - if reply is None: - return "" - if isinstance(reply, str): - return reply - if isinstance(reply, dict): - body = reply.get("msgBody") or reply.get("content", "") - if isinstance(body, dict): - return body.get("content", "") - return str(body) - return str(reply) diff --git a/backend/cli/run.py b/backend/cli/run.py index ba58c81..05d3306 100644 --- a/backend/cli/run.py +++ b/backend/cli/run.py @@ -20,7 +20,9 @@ def _progress(event: str, data: dict[str, Any]) -> None: console.print(f" 第 {data['round']} 轮回复，耗时 {data['latency_ms']}ms") elif event == "rule_result": style = "green" if data["passed"] else "red" - console.print(f" [{data['rule_type']}] {'通过' if data['passed'] else '失败'} - {data['reason']}", style=style) + console.print( + f" [{data['rule_type']}] {'通过' if data['passed'] else '失败'} - {data['reason']}", style=style + ) elif event == "error": console.print(f"执行异常: {data['error']}", style="red") diff --git a/docs/plan-v0.3.md b/docs/plan-v0.3.md new file mode 100644 index 0000000..0f64cfc --- /dev/null +++ b/docs/plan-v0.3.md @@ -0,0 +1,195 @@ +# AgentEvalTool v0.3 「拓」迭代规划 + +**版本**: v0.3.0（规划） +**制定日期**: 2026-07-16 +**基线**: v0.2.0-dev（已部署 t480，含文件管理 + 布局统一 + 6 bug 修复） +**代号**: 「拓」(Extensibility milestone) +**状态**: 待评审 + +--- + +## 一、版本定位 + +v0.3 是「先稳后拓」战略的第二步。v0.2 已让核心闭环具备生产可用性，v0.3 的使命是： + +> 把「单通道 + 3 规则」的稳定闭环，扩展成「多通道 + 可组合规则」的评估平台， +> 并打通 OpenClaw 双向集成，让评估能力可被 AI 助手自主编排。 + +``` +v0.1 (07-09) MVP 闭环 +v1.1 (07-10) Ant Design 前端重构 +v0.2 (07-14) 「稳」async + 安全 + 测试 + 部署 + └─ (07-16) 文件管理 + 布局统一 + bug 修复 +v0.3 (规划) 「拓」← 本文档 + └─ 多通道 · 规则扩展 · OpenClaw 深度集成 · 报告升级 +``` + +--- + +## 二、现状评估 + +### 2.1 已具备的能力 + +| 层面 | 现状 | +|---|---| +| 引擎 | async EvalEngine，真取消，可配置超时，并发信号量 | +| 通道 | 仅 `tutu-api`（工厂已预留 register 扩展点） | +| 规则 | 3 种：keyword_match / response_time / llm_score | +| 报告 | JSON + HTML | +| 存储 | SQLite + Alembic + 级联删除 + 文件管理 | +| 安全 | X-API-Key + .env + CORS 收紧 | +| 测试 | 24 个后端测试，57% 覆盖率 | +| 部署 | 一键脚本 + 镜像版本 tag + /api/health 校验 | + +### 2.2 技术债务盘点（v0.3 前置或并行处理） + +| 编号 | 债务 | 严重度 | 影响 | +|---|---|---|---| +| **DEBT-1** | **规则评估层同步阻塞**：`EvalRule.evaluate()` 是 `def` 而非 `async def`；`llm_score` 用同步 `requests.post(timeout=60)` | 🔴 架构级 | 阻塞事件循环，与 v0.2 async 化目标矛盾；**是 P0 规则扩展的前置阻塞项** | +| DEBT-2 | 后端 `_extract_text`/`_extract_reply_text` 重复 5 处，`_extract_content_from_api_response` 重复 2 处 | 🟡 | 一处修复遗漏另一处即产生 bug | +| DEBT-3 | 前端 `PageWrapper` 已成死代码，6 页面重复布局 JSX；`statusColor` 多页重复 | 🟡 | 改布局需改 6 处 | +| DEBT-4 | 测试盲区：rules 覆盖率 15-30%，文件管理零测试，`_generate_messages` 无测试 | 🟡 | 规则/文件功能回归无保护 | +| DEBT-5 | WebSocket 断线无自动重连；前端 bundle ~2.8MB | 🟢 | 体验问题 | + +> **关键判断**：DEBT-1 必须在 P0 规则扩展之前解决。semantic_similarity（调 embedding API）、safety（调 moderation API）本质都是网络 IO 规则，若继续在同步 `evaluate()` 上叠加，每条规则都会阻塞整个事件循环，v0.2 的异步化收益被规则层抵消。 + +--- + +## 三、v0.3 范围 + +### 3.1 做什么（In Scope） + +#### P0 — 核心能力（必须交付） + +**P0-0 规则评估层异步化**（前置技术改造） +- `EvalRule.evaluate()` → `async def evaluate()` +- `engine._save_rule_results` 相应 `await rule.evaluate(...)` +- `llm_score._call_llm`：`requests` → `httpx.AsyncClient` +- 涉及文件：`evaluation/rules/base.py`、`keyword.py`、`response_time.py`、`llm_score.py`、`evaluation/engine.py`、`tests/unit/test_engine.py` +- 验收：4 个规则全部 async；`pytest` 全绿；并发评测时 LLM 评分不阻塞 WebSocket 推送 + +**P0-1 多通道插件化** +- 新增 `HttpChannel`（通用 HTTP 请求/响应通道，配置化 request 模板 + response JSONPath 提取） +- 新增 `OpenClawChannel`（直接对接 OpenClaw WS，复用现有 proxy 认证逻辑） +- `ChannelFactory` 补齐 `HTTP` / `OPENCLAW` 映射 +- （可选）entry_points 插件发现 —— 见决策点 D3 +- 涉及文件：`channels/http.py`（新）、`channels/openclaw.py`（新）、`channels/factory.py`、`models.py`（ChannelType 已有枚举） +- 验收：可创建 HTTP 通道类型的评测对象并跑通一次评测；连通性测试可用 + +**P0-2 规则扩展 + 组合逻辑** +- 新增规则： + - `semantic_similarity`（回复与参考答案的语义相似度，embedding + 余弦） + - `json_schema`（回复 JSON 结构校验） + - `safety`（敏感内容检测，可接 moderation API 或关键词黑名单降级） +- 组合逻辑：Case 支持 `rule_logic: "all" | "any" | "weighted"`，weighted 支持每规则权重 + 阈值 +- 涉及文件：`rules/semantic.py`（新）、`rules/json_schema.py`（新）、`rules/safety.py`（新）、`rules/__init__.py`、`models.py`（Case 加 rule_logic 字段）、`engine.py`（组合判定逻辑）、Alembic 迁移 +- 验收：3 个新规则注册可用；weighted 组合的 pass_rate 计算正确；前端场景编辑器模板包含新规则示例 + +#### P1 — 重要能力 + +**P1-1 OpenClaw 深度集成（双向）** +- OpenClaw skill 可触发评测（现有 CLI 调用链打通 + 结构化返回） +- 评测完成 webhook 通知（`POST` 到配置的回调地址，附报告摘要） +- 涉及文件：`plugins/openclaw/agenteval_skill.py`、`web/routers/runs.py`（webhook hook）、`config/settings.py`（webhook 配置） +- 验收：OpenClaw 对话触发评测后能收到完成通知 + +**P1-2 报告能力升级** +- 对比报告：选两次 run，side-by-side 展示规则/用例差异 +- Markdown 导出（补充现有 JSON/HTML） +- 报告模板外置（Jinja2 模板从代码抽到 `templates/` 目录） +- 涉及文件：`web/routers/reports.py`、`evaluation/report.py`、`templates/`（新）、前端 `pages/Reports.tsx` +- 验收：对比视图可用；Markdown 导出格式正确 + +**P1-3 测试补全**（还 DEBT-4） +- rules 单测：每个规则（含 3 个新规则）覆盖 pass/fail/边界 +- 文件管理测试：上传/下载/分类级联删除 +- 前端引入 vitest：`sessionReducer` 单测（v0.2 已抽纯函数，就等测试） +- 验收：后端覆盖率 → 70%+；前端有首批 reducer 测试 + +#### P2 — 体验优化 + +**P2-1 场景能力增强** +- 内置场景模板库（单轮问答 / 多轮对话 / 压力测试 / 动态生成） +- 参数化场景（变量占位 + 批量实例化） + +**P2-2 前端体验 + 债务清理**（还 DEBT-3 / DEBT-5） +- WebSocket 自动重连（指数退避） +- `PageWrapper` 复用改造 或 删除；`statusColor` 抽到 tokens +- bundle 优化：路由级懒加载 + +### 3.2 不做什么（Out of Scope） + +明确排除，维持「个人/小团队」定位，避免过度设计： + +- ❌ 多租户 / RBAC 权限系统 +- ❌ PostgreSQL / 分布式数据库（SQLite 足够） +- ❌ Celery / RQ 任务队列（async + asyncio.Task 足够） +- ❌ 移动端适配 +- ❌ entry_points 全动态插件市场（见决策点 D3，倾向轻量注册） + +--- + +## 四、迭代节奏 + +按「先还债、再扩展、后体验」组织为 4 个 Sprint。工作量按人天（PD）估算，基于项目快速迭代节奏。 + +| Sprint | 主题 | 工作项 | 估算 | +|---|---|---|---| +| **S1** | 还债 + 通道 | P0-0 规则异步化 + P0-1 多通道 + DEBT-2 后端工具函数合并 | 4-5 PD | +| **S2** | 规则扩展 | P0-2 新规则 + 组合逻辑 + P1-3 规则测试同步 | 5-6 PD | +| **S3** | 集成 + 报告 | P1-1 OpenClaw 双向 + P1-2 报告升级 | 4-5 PD | +| **S4** | 体验 + 收尾 | P2-1 场景模板 + P2-2 前端债务清理 + 文件管理测试 | 3-4 PD | + +**合计约 16-20 PD（≈ 3-4 周）**，与原 release notes 的 4-5 周估算基本吻合（本计划把还债前置，S1 会略慢）。 + +**里程碑验收点**： +- M1（S1 末）：HTTP 通道跑通一次评测，规则层全 async，pytest 全绿 +- M2（S2 末）：5 种规则 + weighted 组合可用，后端覆盖率 65%+ +- M3（S3 末）：OpenClaw 对话触发评测 + 收到 webhook，对比报告可用 +- M4（S4 末）：场景模板库 + WebSocket 重连，v0.3 部署 t480 + +--- + +## 五、需要拍板的决策点 + +以下决策会显著影响实现，建议在 S1 启动前确认： + +| 编号 | 决策 | 选项 | 倾向 | +|---|---|---|---| +| **D1** | semantic_similarity 的 embedding 来源 | A) 外部 API（豆包/OpenAI embedding，成本+网络依赖） B) 本地 sentence-transformers（部署重，+模型体积） | **A**：与现有 llm_score 一致的外部 API 模式，部署轻 | +| **D2** | safety 规则实现深度 | A) 接 moderation API B) 关键词黑名单 C) 两者，API 不可用时降级黑名单 | **C**：默认黑名单保证可用，API 可选增强 | +| **D3** | 通道/规则插件机制 | A) entry_points 动态发现 B) 现有工厂 register + 装饰器注册 | **B**：小团队无需插件市场，现有机制已够，避免过度设计 | +| **D4** | 规则异步化是否 breaking | 改 `evaluate` 签名会影响所有规则 | 全量改造（4 规则少，一次到位），无需兼容层 | +| **D5** | v0.3 是否顺带升级前端数据层（TanStack Query） | A) 引入 B) 维持 useState + 手动 load | **B**：当前页面数据量小，S4 仅做重连 + 懒加载，Query 留 v0.4 | + +--- + +## 六、风险 + +| 风险 | 影响 | 缓解 | +|---|---|---| +| 规则异步化牵连 engine + 全部规则 + 测试 | S1 工期 | 规则数量少（4 个），且有 24 个测试兜底，一次改到位 | +| semantic/safety 依赖外部 API 稳定性 | 评测可靠性 | 规则内 try/except，API 失败降级为「规则跳过 + 明确 reason」，不中止 run | +| OpenClaw webhook 依赖外部可达性 | P1-1 集成 | webhook 失败不影响评测结果，仅记录日志 | +| 组合逻辑 weighted 的 pass_rate 语义变化 | 报告一致性 | 明确定义：weighted 下 case 通过 = 加权分 ≥ 阈值；文档 + 测试锁定 | + +--- + +## 七、验收标准（v0.3 Definition of Done） + +1. 可创建 tutu-api / HTTP / OpenClaw 三类通道的评测对象并各跑通一次评测 +2. 5 种规则（3 旧 + semantic/json_schema/safety 选交付）可用，支持 all/any/weighted 组合 +3. 规则评估层全 async，并发评测时 LLM/embedding 调用不阻塞 WebSocket 推送 +4. OpenClaw 对话可触发评测并收到完成通知 +5. 报告支持 JSON/HTML/Markdown 导出 + 两次 run 对比 +6. 后端测试覆盖率 ≥ 70%，前端有首批 reducer 测试 +7. 后端工具函数去重（DEBT-2），前端布局债务清理（DEBT-3） +8. v0.3 部署 t480，`/api/health` 显示 0.3.0 版本 + +--- + +## 八、下一步 + +1. 评审本计划，确认第五节 5 个决策点 +2. 确认后按 Sprint 拆分为可执行 issue（每项含涉及文件 + 验收） +3. 从 S1（规则异步化 + 多通道）启动 diff --git a/tests/unit/test_http_channel_and_rules.py b/tests/unit/test_http_channel_and_rules.py new file mode 100644 index 0000000..315c2ce --- /dev/null +++ b/tests/unit/test_http_channel_and_rules.py @@ -0,0 +1,213 @@ +"""Unit tests for HttpChannel and async rule evaluation.""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from agenteval.channels.http import HttpChannel, _get_path +from agenteval.channels.base import SendResult +from agenteval.evaluation.rules.keyword import KeywordMatchRule +from agenteval.evaluation.rules.response_time import ResponseTimeRule +from agenteval.models import Case, CaseType, Expectation, Turn + + +# ── _get_path helper ───────────────────────────────────────────────────── + +def test_get_path_simple(): + assert _get_path({"id": "abc"}, "id") == "abc" + + +def test_get_path_nested(): + assert _get_path({"a": {"b": {"c": 42}}}, "a.b.c") == 42 + + +def test_get_path_missing(): + assert _get_path({"a": 1}, "a.b") is None + + +def test_get_path_none_data(): + assert _get_path(None, "x") is None + + +def test_get_path_list_index(): + assert _get_path({"items": ["x", "y"]}, "items.1") == "y" + + +# ── HttpChannel ────────────────────────────────────────────────────────── + +def _make_channel(**extra) -> HttpChannel: + config = { + "send_url": "http://mock/send", + "reply_url": "http://mock/reply/{msg_id}", + "health_url": "http://mock/health", + **extra, + } + return HttpChannel(config) + + +async def test_http_health_check_ok(): + ch = _make_channel() + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.raise_for_status = MagicMock() + with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)): + result = await ch.health_check() + assert result.ok is True + assert "200" in result.message + + +async def test_http_health_check_fail(): + ch = _make_channel() + with patch.object(ch._client, "get", new=AsyncMock(side_effect=Exception("conn refused"))): + result = await ch.health_check() + assert result.ok is False + assert "conn refused" in result.message + + +async def test_http_send_extracts_msg_id(): + ch = _make_channel(msg_id_path="data.id") + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.json = MagicMock(return_value={"data": {"id": "msg-42"}}) + with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)): + result = await ch.send("hello") + assert result.ok is True + assert result.question_msg_id == "msg-42" + + +async def test_http_send_failure(): + ch = _make_channel() + with patch.object(ch._client, "post", new=AsyncMock(side_effect=Exception("timeout"))): + result = await ch.send("hi") + assert result.ok is False + assert "timeout" in result.error + + +async def test_http_poll_reply_found(): + ch = _make_channel(reply_path="answer") + call_count = {"n": 0} + + async def mock_get(url, **kwargs): + call_count["n"] += 1 + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.json = MagicMock(return_value={"answer": "Hello world"}) + return mock_resp + + with patch.object(ch._client, "get", new=mock_get): + reply = await ch.poll_reply("msg-1", timeout=5.0) + assert reply is not None + assert reply.content == "Hello world" + assert reply.question_msg_id == "msg-1" + + +async def test_http_poll_reply_timeout(): + ch = _make_channel(reply_path="missing_field", poll_interval=0.05) + + async def mock_get(url, **kwargs): + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.json = MagicMock(return_value={}) # field not present + return mock_resp + + with patch.object(ch._client, "get", new=mock_get): + reply = await ch.poll_reply("msg-1", timeout=0.15, poll_interval=0.05) + assert reply is None + + +async def test_http_poll_reply_readiness_flag(): + ch = _make_channel(reply_path="text", reply_ready_path="ready") + responses = [ + {"ready": False, "text": "not ready"}, + {"ready": True, "text": "final answer"}, + ] + call_idx = {"n": 0} + + async def mock_get(url, **kwargs): + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.json = MagicMock(return_value=responses[min(call_idx["n"], 1)]) + call_idx["n"] += 1 + return mock_resp + + with patch.object(ch._client, "get", new=mock_get): + reply = await ch.poll_reply("msg-1", timeout=5.0, poll_interval=0.05) + assert reply is not None + assert reply.content == "final answer" + + +# ── Async rule evaluation ──────────────────────────────────────────────── + +def _make_turn(reply_text: str, latency_ms: int = 100) -> Turn: + return Turn( + id="t1", run_id="r1", case_id="c1", round_index=1, + reply={"msgBody": {"content": reply_text}}, + latency_ms=latency_ms, + ) + + +def _make_case(*, keywords: list[str] | None = None, max_ms: int | None = None) -> Case: + return Case( + id="c1", type=CaseType.SINGLE, messages=["hi"], + expectations=Expectation( + keywords_include=keywords or [], + response_time_max_ms=max_ms, + ), + ) + + +async def test_keyword_rule_is_async_and_passes(): + rule = KeywordMatchRule({"keywords": ["hello", "world"]}) + turn = _make_turn("hello world here") + result = await rule.evaluate(_make_case(), [turn]) + assert result.passed is True + assert result.score == 1.0 + + +async def test_keyword_rule_fails_missing_keyword(): + rule = KeywordMatchRule({"keywords": ["missing"]}) + turn = _make_turn("some other text") + result = await rule.evaluate(_make_case(), [turn]) + assert result.passed is False + assert "missing" in result.reason + + +async def test_keyword_rule_fails_excluded_keyword(): + rule = KeywordMatchRule({"exclude_keywords": ["banned"]}) + turn = _make_turn("this is banned content") + result = await rule.evaluate(_make_case(), [turn]) + assert result.passed is False + + +async def test_response_time_rule_is_async_and_passes(): + rule = ResponseTimeRule({"max_ms": 500}) + turn = _make_turn("ok", latency_ms=200) + result = await rule.evaluate(_make_case(), [turn]) + assert result.passed is True + + +async def test_response_time_rule_fails_over_threshold(): + rule = ResponseTimeRule({"max_ms": 100}) + turn = _make_turn("slow response", latency_ms=5000) + result = await rule.evaluate(_make_case(), [turn]) + assert result.passed is False + assert "5000ms" in result.reason + + +async def test_response_time_uses_expectation_fallback(): + rule = ResponseTimeRule({}) # no max_ms in params + case = _make_case(max_ms=200) + turn = _make_turn("ok", latency_ms=100) + result = await rule.evaluate(case, [turn]) + assert result.passed is True + + +async def test_rules_empty_dialog_fail(): + keyword_rule = KeywordMatchRule({"keywords": ["x"]}) + rt_rule = ResponseTimeRule({"max_ms": 1000}) + case = _make_case() + for rule in [keyword_rule, rt_rule]: + result = await rule.evaluate(case, []) + assert result.passed is False + assert "无回复" in result.reason