c956da7686
All checks were successful
CI / test (pull_request) Successful in 4m3s
- token 用量接入：ModelGateway 经 adapter.parse_usage 累计评测侧 LLM 调用的 token 消耗，引擎写入 run.summary.eval_token_usage，报告透出 - 放弃率落地：CaseOutcome 新增 abandoned 标记（对话中途发送/接收失败）， build_run_summary 统计 abandoned_cases / abandonment_rate - Go/No-Go 可配置：Scenario 新增 acceptance_criteria 字段（DB 列 + 幂等迁移）， 报告按场景标准出 verdict，缺省回退全局默认；标准变更不触发考纲升版
101 lines
4.1 KiB
Python
101 lines
4.1 KiB
Python
"""Shared multi-protocol model transport used by evaluation features."""
|
||
|
||
from typing import Any
|
||
|
||
import httpx
|
||
|
||
from agenteval.model_protocols import ModelProtocolAdapter, ProtocolAdapterError, get_protocol_adapter
|
||
from agenteval.services.model_configs import ModelRuntimeConfig
|
||
|
||
|
||
class ModelGatewayError(RuntimeError):
|
||
pass
|
||
|
||
|
||
class ModelGateway:
|
||
def __init__(self, timeout: float = 60.0, transport: httpx.AsyncBaseTransport | None = None):
|
||
self.timeout = timeout
|
||
self.transport = transport
|
||
# 评测侧 LLM 调用的累计 token 用量（引擎结束时写入 run summary）
|
||
self.total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
||
|
||
def _record_usage(self, adapter: ModelProtocolAdapter, data: dict[str, Any]) -> None:
|
||
usage = adapter.parse_usage(data)
|
||
if not usage:
|
||
return
|
||
for key in self.total_usage:
|
||
self.total_usage[key] += int(usage.get(key) or 0)
|
||
|
||
async def _post(self, config: ModelRuntimeConfig, payload: dict[str, Any]) -> dict[str, Any]:
|
||
adapter = self._adapter(config)
|
||
try:
|
||
async with httpx.AsyncClient(timeout=self.timeout, transport=self.transport) as client:
|
||
response = await client.post(
|
||
config.endpoint_url,
|
||
headers=adapter.headers(config.api_key),
|
||
json=payload,
|
||
)
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
if not isinstance(data, dict):
|
||
raise ModelGatewayError("模型接口返回格式不是 JSON 对象")
|
||
return data
|
||
except ModelGatewayError:
|
||
raise
|
||
except httpx.HTTPStatusError as exc:
|
||
detail = exc.response.text[:300]
|
||
raise ModelGatewayError(f"模型接口返回 HTTP {exc.response.status_code}: {detail}") from exc
|
||
except Exception as exc:
|
||
raise ModelGatewayError(f"模型接口调用失败: {exc}") from exc
|
||
|
||
@staticmethod
|
||
def _adapter(config: ModelRuntimeConfig):
|
||
try:
|
||
return get_protocol_adapter(config.provider)
|
||
except ProtocolAdapterError as exc:
|
||
raise ModelGatewayError(str(exc)) from exc
|
||
|
||
async def chat(
|
||
self,
|
||
config: ModelRuntimeConfig,
|
||
messages: list[dict[str, str]],
|
||
temperature: float = 0.2,
|
||
) -> str:
|
||
adapter = self._adapter(config)
|
||
try:
|
||
payload = adapter.chat_payload(config.model_name, messages, temperature)
|
||
data = await self._post(config, payload)
|
||
self._record_usage(adapter, data)
|
||
return adapter.parse_chat(data)
|
||
except ProtocolAdapterError as exc:
|
||
raise ModelGatewayError(str(exc)) from exc
|
||
|
||
async def embed(self, config: ModelRuntimeConfig, inputs: str | list[str]) -> list[list[float]]:
|
||
adapter = self._adapter(config)
|
||
try:
|
||
payload = adapter.embedding_payload(config.model_name, inputs)
|
||
data = await self._post(config, payload)
|
||
self._record_usage(adapter, data)
|
||
return adapter.parse_embeddings(data)
|
||
except ProtocolAdapterError as exc:
|
||
raise ModelGatewayError(str(exc)) from exc
|
||
|
||
async def moderate(self, config: ModelRuntimeConfig, text: str) -> dict[str, Any]:
|
||
adapter = self._adapter(config)
|
||
try:
|
||
payload = adapter.moderation_payload(config.model_name, text)
|
||
return adapter.parse_moderation(await self._post(config, payload))
|
||
except ProtocolAdapterError as exc:
|
||
raise ModelGatewayError(str(exc)) from exc
|
||
|
||
async def test_connection(self, config: ModelRuntimeConfig) -> str:
|
||
if config.capability.value == "chat":
|
||
await self.chat(config, [{"role": "user", "content": "回复 OK"}], temperature=0)
|
||
elif config.capability.value == "embedding":
|
||
vectors = await self.embed(config, "connection test")
|
||
if not vectors or not vectors[0]:
|
||
raise ModelGatewayError("Embedding 接口未返回向量")
|
||
else:
|
||
await self.moderate(config, "connection test")
|
||
return "连接成功"
|