c956da7686
All checks were successful
CI / test (pull_request) Successful in 4m3s
- token 用量接入：ModelGateway 经 adapter.parse_usage 累计评测侧 LLM 调用的 token 消耗，引擎写入 run.summary.eval_token_usage，报告透出 - 放弃率落地：CaseOutcome 新增 abandoned 标记（对话中途发送/接收失败）， build_run_summary 统计 abandoned_cases / abandonment_rate - Go/No-Go 可配置：Scenario 新增 acceptance_criteria 字段（DB 列 + 幂等迁移）， 报告按场景标准出 verdict，缺省回退全局默认；标准变更不触发考纲升版
54 lines
1.8 KiB
Python
54 lines
1.8 KiB
Python
"""Protocol adapter contract for external model APIs."""
|
|
|
|
from typing import Any
|
|
|
|
from agenteval.models import ModelCapability, ModelProtocol
|
|
|
|
|
|
class ProtocolAdapterError(ValueError):
|
|
pass
|
|
|
|
|
|
class ModelProtocolAdapter:
|
|
protocol: ModelProtocol
|
|
supported_capabilities: frozenset[ModelCapability] = frozenset()
|
|
|
|
def headers(self, api_key: str | None) -> dict[str, str]:
|
|
return {"Content-Type": "application/json"}
|
|
|
|
def chat_payload(
|
|
self,
|
|
model_name: str | None,
|
|
messages: list[dict[str, str]],
|
|
temperature: float,
|
|
) -> dict[str, Any]:
|
|
self._unsupported(ModelCapability.CHAT)
|
|
|
|
def parse_chat(self, data: dict[str, Any]) -> str:
|
|
self._unsupported(ModelCapability.CHAT)
|
|
|
|
def embedding_payload(self, model_name: str | None, inputs: str | list[str]) -> dict[str, Any]:
|
|
self._unsupported(ModelCapability.EMBEDDING)
|
|
|
|
def parse_embeddings(self, data: dict[str, Any]) -> list[list[float]]:
|
|
self._unsupported(ModelCapability.EMBEDDING)
|
|
|
|
def moderation_payload(self, model_name: str | None, text: str) -> dict[str, Any]:
|
|
self._unsupported(ModelCapability.MODERATION)
|
|
|
|
def parse_moderation(self, data: dict[str, Any]) -> dict[str, Any]:
|
|
self._unsupported(ModelCapability.MODERATION)
|
|
|
|
def parse_usage(self, data: dict[str, Any]) -> dict[str, int] | None:
|
|
"""Extract token usage from a response; None for protocols that omit it."""
|
|
return None
|
|
|
|
def _unsupported(self, capability: ModelCapability) -> None:
|
|
raise ProtocolAdapterError(f"{self.protocol.value} 协议不支持 {capability.value} 能力")
|
|
|
|
@staticmethod
|
|
def require_model(model_name: str | None) -> str:
|
|
if not model_name:
|
|
raise ProtocolAdapterError("模型名称不能为空")
|
|
return model_name
|