diff --git a/backend/agenteval/model_gateway.py b/backend/agenteval/model_gateway.py index 0967d81..67d6456 100644 --- a/backend/agenteval/model_gateway.py +++ b/backend/agenteval/model_gateway.py @@ -1,11 +1,11 @@ -"""Shared OpenAI-compatible model transport used by evaluation features.""" +"""Shared multi-protocol model transport used by evaluation features.""" from typing import Any import httpx +from agenteval.model_protocols import ProtocolAdapterError, get_protocol_adapter from agenteval.services.model_configs import ModelRuntimeConfig -from agenteval.utils.llm import extract_content_from_llm_response class ModelGatewayError(RuntimeError): @@ -13,22 +13,17 @@ class ModelGatewayError(RuntimeError): class ModelGateway: - def __init__(self, timeout: float = 60.0): + def __init__(self, timeout: float = 60.0, transport: httpx.AsyncBaseTransport | None = None): self.timeout = timeout - - @staticmethod - def _headers(config: ModelRuntimeConfig) -> dict[str, str]: - headers = {"Content-Type": "application/json"} - if config.api_key: - headers["Authorization"] = f"Bearer {config.api_key}" - return headers + self.transport = transport async def _post(self, config: ModelRuntimeConfig, payload: dict[str, Any]) -> dict[str, Any]: + adapter = self._adapter(config) try: - async with httpx.AsyncClient(timeout=self.timeout) as client: + async with httpx.AsyncClient(timeout=self.timeout, transport=self.transport) as client: response = await client.post( config.endpoint_url, - headers=self._headers(config), + headers=adapter.headers(config.api_key), json=payload, ) response.raise_for_status() @@ -44,42 +39,41 @@ class ModelGateway: except Exception as exc: raise ModelGatewayError(f"模型接口调用失败: {exc}") from exc + @staticmethod + def _adapter(config: ModelRuntimeConfig): + try: + return get_protocol_adapter(config.provider) + except ProtocolAdapterError as exc: + raise ModelGatewayError(str(exc)) from exc + async def chat( self, config: ModelRuntimeConfig, messages: list[dict[str, str]], temperature: float = 0.2, ) -> str: - payload = { - "model": config.model_name, - "messages": messages, - "temperature": temperature, - } - content = extract_content_from_llm_response(await self._post(config, payload)) - if not content: - raise ModelGatewayError("模型接口返回内容为空") - return content + adapter = self._adapter(config) + try: + payload = adapter.chat_payload(config.model_name, messages, temperature) + return adapter.parse_chat(await self._post(config, payload)) + except ProtocolAdapterError as exc: + raise ModelGatewayError(str(exc)) from exc async def embed(self, config: ModelRuntimeConfig, inputs: str | list[str]) -> list[list[float]]: - data = await self._post(config, {"model": config.model_name, "input": inputs}) + adapter = self._adapter(config) try: - rows = sorted(data["data"], key=lambda item: item.get("index", 0)) - return [row["embedding"] for row in rows] - except (KeyError, TypeError) as exc: - raise ModelGatewayError("Embedding 接口返回格式不正确") from exc + payload = adapter.embedding_payload(config.model_name, inputs) + return adapter.parse_embeddings(await self._post(config, payload)) + except ProtocolAdapterError as exc: + raise ModelGatewayError(str(exc)) from exc async def moderate(self, config: ModelRuntimeConfig, text: str) -> dict[str, Any]: - payload: dict[str, Any] = {"input": text} - if config.model_name: - payload["model"] = config.model_name - data = await self._post(config, payload) + adapter = self._adapter(config) try: - result = data["results"][0] - except (KeyError, IndexError, TypeError) as exc: - raise ModelGatewayError("Moderation 接口返回格式不正确") from exc - if not isinstance(result, dict): - raise ModelGatewayError("Moderation 接口返回格式不正确") - return result + payload = adapter.moderation_payload(config.model_name, text) + return adapter.parse_moderation(await self._post(config, payload)) + except ProtocolAdapterError as exc: + raise ModelGatewayError(str(exc)) from exc async def test_connection(self, config: ModelRuntimeConfig) -> str: if config.capability.value == "chat": diff --git a/backend/agenteval/model_protocols/__init__.py b/backend/agenteval/model_protocols/__init__.py new file mode 100644 index 0000000..8c22142 --- /dev/null +++ b/backend/agenteval/model_protocols/__init__.py @@ -0,0 +1,39 @@ +"""Registry of supported external model protocols.""" + +from agenteval.models import ModelCapability, ModelProtocol + +from .anthropic import AnthropicAdapter +from .base import ModelProtocolAdapter, ProtocolAdapterError +from .dashscope import DashScopeAdapter +from .gemini import GoogleGeminiAdapter +from .openai import OpenAICompatibleAdapter + +_ADAPTERS: dict[ModelProtocol, ModelProtocolAdapter] = { + adapter.protocol: adapter + for adapter in ( + OpenAICompatibleAdapter(), + AnthropicAdapter(), + GoogleGeminiAdapter(), + DashScopeAdapter(), + ) +} + + +def get_protocol_adapter(provider: str | ModelProtocol) -> ModelProtocolAdapter: + try: + protocol = ModelProtocol(provider) + return _ADAPTERS[protocol] + except (ValueError, KeyError) as exc: + raise ProtocolAdapterError(f"不支持的模型协议: {provider}") from exc + + +def get_protocol_capabilities(provider: str | ModelProtocol) -> frozenset[ModelCapability]: + return get_protocol_adapter(provider).supported_capabilities + + +__all__ = [ + "ModelProtocolAdapter", + "ProtocolAdapterError", + "get_protocol_adapter", + "get_protocol_capabilities", +] diff --git a/backend/agenteval/model_protocols/anthropic.py b/backend/agenteval/model_protocols/anthropic.py new file mode 100644 index 0000000..38a21de --- /dev/null +++ b/backend/agenteval/model_protocols/anthropic.py @@ -0,0 +1,57 @@ +"""Anthropic Messages API protocol.""" + +from typing import Any + +from agenteval.models import ModelCapability, ModelProtocol + +from .base import ModelProtocolAdapter, ProtocolAdapterError + + +class AnthropicAdapter(ModelProtocolAdapter): + protocol = ModelProtocol.ANTHROPIC + supported_capabilities = frozenset({ModelCapability.CHAT}) + + def headers(self, api_key: str | None) -> dict[str, str]: + headers = super().headers(api_key) + headers["anthropic-version"] = "2023-06-01" + if api_key: + headers["x-api-key"] = api_key + return headers + + def chat_payload( + self, + model_name: str | None, + messages: list[dict[str, str]], + temperature: float, + ) -> dict[str, Any]: + system_parts: list[str] = [] + api_messages: list[dict[str, str]] = [] + for message in messages: + role = message.get("role", "user") + content = message.get("content", "") + if role in {"system", "developer"}: + system_parts.append(content) + else: + api_messages.append({"role": "assistant" if role == "assistant" else "user", "content": content}) + payload: dict[str, Any] = { + "model": self.require_model(model_name), + "max_tokens": 1024, + "messages": api_messages, + "temperature": temperature, + } + if system_parts: + payload["system"] = "\n\n".join(system_parts) + return payload + + def parse_chat(self, data: dict[str, Any]) -> str: + blocks = data.get("content") + if not isinstance(blocks, list): + raise ProtocolAdapterError("Anthropic 接口返回格式不正确") + content = "\n".join( + str(block.get("text", "")) + for block in blocks + if isinstance(block, dict) and block.get("type") == "text" and block.get("text") + ) + if not content: + raise ProtocolAdapterError("模型接口返回内容为空") + return content diff --git a/backend/agenteval/model_protocols/base.py b/backend/agenteval/model_protocols/base.py new file mode 100644 index 0000000..b9ed429 --- /dev/null +++ b/backend/agenteval/model_protocols/base.py @@ -0,0 +1,49 @@ +"""Protocol adapter contract for external model APIs.""" + +from typing import Any + +from agenteval.models import ModelCapability, ModelProtocol + + +class ProtocolAdapterError(ValueError): + pass + + +class ModelProtocolAdapter: + protocol: ModelProtocol + supported_capabilities: frozenset[ModelCapability] = frozenset() + + def headers(self, api_key: str | None) -> dict[str, str]: + return {"Content-Type": "application/json"} + + def chat_payload( + self, + model_name: str | None, + messages: list[dict[str, str]], + temperature: float, + ) -> dict[str, Any]: + self._unsupported(ModelCapability.CHAT) + + def parse_chat(self, data: dict[str, Any]) -> str: + self._unsupported(ModelCapability.CHAT) + + def embedding_payload(self, model_name: str | None, inputs: str | list[str]) -> dict[str, Any]: + self._unsupported(ModelCapability.EMBEDDING) + + def parse_embeddings(self, data: dict[str, Any]) -> list[list[float]]: + self._unsupported(ModelCapability.EMBEDDING) + + def moderation_payload(self, model_name: str | None, text: str) -> dict[str, Any]: + self._unsupported(ModelCapability.MODERATION) + + def parse_moderation(self, data: dict[str, Any]) -> dict[str, Any]: + self._unsupported(ModelCapability.MODERATION) + + def _unsupported(self, capability: ModelCapability) -> None: + raise ProtocolAdapterError(f"{self.protocol.value} 协议不支持 {capability.value} 能力") + + @staticmethod + def require_model(model_name: str | None) -> str: + if not model_name: + raise ProtocolAdapterError("模型名称不能为空") + return model_name diff --git a/backend/agenteval/model_protocols/dashscope.py b/backend/agenteval/model_protocols/dashscope.py new file mode 100644 index 0000000..1cd7cc3 --- /dev/null +++ b/backend/agenteval/model_protocols/dashscope.py @@ -0,0 +1,45 @@ +"""Alibaba Cloud Model Studio native DashScope protocol.""" + +from typing import Any + +from agenteval.models import ModelCapability, ModelProtocol + +from .base import ModelProtocolAdapter, ProtocolAdapterError + + +class DashScopeAdapter(ModelProtocolAdapter): + protocol = ModelProtocol.DASHSCOPE + supported_capabilities = frozenset({ModelCapability.CHAT}) + + def headers(self, api_key: str | None) -> dict[str, str]: + headers = super().headers(api_key) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + def chat_payload( + self, + model_name: str | None, + messages: list[dict[str, str]], + temperature: float, + ) -> dict[str, Any]: + return { + "model": self.require_model(model_name), + "input": {"messages": messages}, + "parameters": {"result_format": "message", "temperature": temperature}, + } + + def parse_chat(self, data: dict[str, Any]) -> str: + try: + content = data["output"]["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError) as exc: + raise ProtocolAdapterError("DashScope 接口返回格式不正确") from exc + if isinstance(content, list): + content = "\n".join( + str(block.get("text") or block.get("content") or "") + for block in content + if isinstance(block, dict) + ) + if not content: + raise ProtocolAdapterError("模型接口返回内容为空") + return str(content) diff --git a/backend/agenteval/model_protocols/gemini.py b/backend/agenteval/model_protocols/gemini.py new file mode 100644 index 0000000..772ff17 --- /dev/null +++ b/backend/agenteval/model_protocols/gemini.py @@ -0,0 +1,56 @@ +"""Google Gemini generateContent REST protocol.""" + +from typing import Any + +from agenteval.models import ModelCapability, ModelProtocol + +from .base import ModelProtocolAdapter, ProtocolAdapterError + + +class GoogleGeminiAdapter(ModelProtocolAdapter): + protocol = ModelProtocol.GOOGLE_GEMINI + supported_capabilities = frozenset({ModelCapability.CHAT}) + + def headers(self, api_key: str | None) -> dict[str, str]: + headers = super().headers(api_key) + if api_key: + headers["x-goog-api-key"] = api_key + return headers + + def chat_payload( + self, + model_name: str | None, + messages: list[dict[str, str]], + temperature: float, + ) -> dict[str, Any]: + self.require_model(model_name) + system_parts: list[dict[str, str]] = [] + contents: list[dict[str, Any]] = [] + for message in messages: + role = message.get("role", "user") + part = {"text": message.get("content", "")} + if role in {"system", "developer"}: + system_parts.append(part) + else: + contents.append({"role": "model" if role == "assistant" else "user", "parts": [part]}) + payload: dict[str, Any] = { + "contents": contents, + "generationConfig": {"temperature": temperature}, + } + if system_parts: + payload["systemInstruction"] = {"parts": system_parts} + return payload + + def parse_chat(self, data: dict[str, Any]) -> str: + try: + parts = data["candidates"][0]["content"]["parts"] + except (KeyError, IndexError, TypeError) as exc: + raise ProtocolAdapterError("Gemini 接口返回格式不正确") from exc + content = "\n".join( + str(part.get("text", "")) + for part in parts + if isinstance(part, dict) and part.get("text") + ) + if not content: + raise ProtocolAdapterError("模型接口返回内容为空") + return content diff --git a/backend/agenteval/model_protocols/openai.py b/backend/agenteval/model_protocols/openai.py new file mode 100644 index 0000000..02f8fd0 --- /dev/null +++ b/backend/agenteval/model_protocols/openai.py @@ -0,0 +1,61 @@ +"""OpenAI-compatible chat, embedding, and moderation protocol.""" + +from typing import Any + +from agenteval.models import ModelCapability, ModelProtocol +from agenteval.utils.llm import extract_content_from_llm_response + +from .base import ModelProtocolAdapter, ProtocolAdapterError + + +class OpenAICompatibleAdapter(ModelProtocolAdapter): + protocol = ModelProtocol.OPENAI_COMPATIBLE + supported_capabilities = frozenset(ModelCapability) + + def headers(self, api_key: str | None) -> dict[str, str]: + headers = super().headers(api_key) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + def chat_payload( + self, + model_name: str | None, + messages: list[dict[str, str]], + temperature: float, + ) -> dict[str, Any]: + return {"model": self.require_model(model_name), "messages": messages, "temperature": temperature} + + def parse_chat(self, data: dict[str, Any]) -> str: + content = extract_content_from_llm_response(data) + if not content: + raise ProtocolAdapterError("模型接口返回内容为空") + return content + + def embedding_payload(self, model_name: str | None, inputs: str | list[str]) -> dict[str, Any]: + return {"model": self.require_model(model_name), "input": inputs} + + def parse_embeddings(self, data: dict[str, Any]) -> list[list[float]]: + try: + rows = sorted(data["data"], key=lambda item: item.get("index", 0)) + vectors = [row["embedding"] for row in rows] + except (KeyError, TypeError) as exc: + raise ProtocolAdapterError("Embedding 接口返回格式不正确") from exc + if not all(isinstance(vector, list) for vector in vectors): + raise ProtocolAdapterError("Embedding 接口返回格式不正确") + return vectors + + def moderation_payload(self, model_name: str | None, text: str) -> dict[str, Any]: + payload: dict[str, Any] = {"input": text} + if model_name: + payload["model"] = model_name + return payload + + def parse_moderation(self, data: dict[str, Any]) -> dict[str, Any]: + try: + result = data["results"][0] + except (KeyError, IndexError, TypeError) as exc: + raise ProtocolAdapterError("Moderation 接口返回格式不正确") from exc + if not isinstance(result, dict): + raise ProtocolAdapterError("Moderation 接口返回格式不正确") + return result diff --git a/backend/agenteval/models.py b/backend/agenteval/models.py index 9faa7d6..fe53cdc 100644 --- a/backend/agenteval/models.py +++ b/backend/agenteval/models.py @@ -36,6 +36,13 @@ class ModelCapability(str, Enum): MODERATION = "moderation" +class ModelProtocol(str, Enum): + OPENAI_COMPATIBLE = "openai_compatible" + ANTHROPIC = "anthropic" + GOOGLE_GEMINI = "google_gemini" + DASHSCOPE = "dashscope" + + class ModelPurpose(str, Enum): GENERATOR = "generator" JUDGE = "judge" diff --git a/backend/agenteval/services/model_configs.py b/backend/agenteval/services/model_configs.py index be64f2f..68146c4 100644 --- a/backend/agenteval/services/model_configs.py +++ b/backend/agenteval/services/model_configs.py @@ -8,6 +8,7 @@ from cryptography.fernet import Fernet, InvalidToken from sqlmodel import Session from agenteval.config import get_settings +from agenteval.model_protocols import ProtocolAdapterError, get_protocol_adapter from agenteval.models import ModelCapability, ModelPurpose from agenteval.storage.db import ModelConfigDB, ScenarioDB from agenteval.storage.model_config_repository import ModelConfigRepository @@ -94,11 +95,17 @@ class ModelConfigService: self.cipher = cipher or SecretCipher() @staticmethod - def validate_fields(capability: str, endpoint_url: str, model_name: str | None) -> None: + def validate_fields(provider: str, capability: str, endpoint_url: str, model_name: str | None) -> None: try: capability_value = ModelCapability(capability) except ValueError as exc: raise ModelConfigError(f"不支持的模型能力: {capability}") from exc + try: + adapter = get_protocol_adapter(provider) + except ProtocolAdapterError as exc: + raise ModelConfigError(str(exc)) from exc + if capability_value not in adapter.supported_capabilities: + raise ModelConfigError(f"{provider} 协议不支持 {capability_value.value} 能力") parsed = urlparse(endpoint_url) if parsed.scheme not in {"http", "https"} or not parsed.netloc: raise ModelConfigError("Endpoint 必须是有效的 HTTP 或 HTTPS URL") @@ -109,8 +116,10 @@ class ModelConfigService: def validate_common(name: str, provider: str, enabled: bool, is_default: bool) -> None: if not name.strip(): raise ModelConfigError("模型配置名称不能为空") - if provider != "openai_compatible": - raise ModelConfigError(f"不支持的模型协议: {provider}") + try: + get_protocol_adapter(provider) + except ProtocolAdapterError as exc: + raise ModelConfigError(str(exc)) from exc if is_default and not enabled: raise ModelConfigError("停用的模型配置不能设为默认") @@ -129,7 +138,7 @@ class ModelConfigService: ) -> ModelConfigDB: name = name.strip() self.validate_common(name, provider, enabled, is_default) - self.validate_fields(capability, endpoint_url.strip(), model_name) + self.validate_fields(provider, capability, endpoint_url.strip(), model_name) if self.repo.get_by_name(name): raise ModelConfigError("模型配置名称已存在") config = ModelConfigDB( @@ -163,7 +172,7 @@ class ModelConfigService: config = self.require(config_id) name = name.strip() self.validate_common(name, provider, enabled, is_default) - self.validate_fields(capability, endpoint_url.strip(), model_name) + self.validate_fields(provider, capability, endpoint_url.strip(), model_name) references = self.repo.list_references(config_id) if references and not enabled: raise ModelConfigError("模型配置正在被场景引用，不能停用") diff --git a/backend/agenteval/web/model_config_schemas.py b/backend/agenteval/web/model_config_schemas.py index 032050f..1cad5bc 100644 --- a/backend/agenteval/web/model_config_schemas.py +++ b/backend/agenteval/web/model_config_schemas.py @@ -4,13 +4,13 @@ from datetime import datetime from pydantic import BaseModel, Field -from agenteval.models import ModelCapability +from agenteval.models import ModelCapability, ModelProtocol from agenteval.storage.db import ModelConfigDB, iso_utc class ModelConfigCreate(BaseModel): name: str = Field(min_length=1, max_length=100) - provider: str = "openai_compatible" + provider: ModelProtocol = ModelProtocol.OPENAI_COMPATIBLE capability: ModelCapability endpoint_url: str model_name: str | None = None @@ -27,7 +27,7 @@ class ModelConfigUpdate(ModelConfigCreate): class ModelConfigResponse(BaseModel): id: str name: str - provider: str + provider: ModelProtocol capability: ModelCapability endpoint_url: str model_name: str | None @@ -43,7 +43,7 @@ class ModelConfigResponse(BaseModel): return cls( id=config.id or "", name=config.name, - provider=config.provider, + provider=ModelProtocol(config.provider), capability=ModelCapability(config.capability), endpoint_url=config.endpoint_url, model_name=config.model_name, diff --git a/frontend/web/src/api.ts b/frontend/web/src/api.ts index 13b099d..8d3ac73 100644 --- a/frontend/web/src/api.ts +++ b/frontend/web/src/api.ts @@ -41,11 +41,12 @@ export interface Scenario { } export type ModelCapability = 'chat' | 'embedding' | 'moderation' +export type ModelProtocol = 'openai_compatible' | 'anthropic' | 'google_gemini' | 'dashscope' export interface ModelConfig { id: string name: string - provider: string + provider: ModelProtocol capability: ModelCapability endpoint_url: string model_name: string | null @@ -59,7 +60,7 @@ export interface ModelConfig { export interface ModelConfigPayload { name: string - provider: string + provider: ModelProtocol capability: ModelCapability endpoint_url: string model_name?: string | null diff --git a/frontend/web/src/pages/ModelConfigs.tsx b/frontend/web/src/pages/ModelConfigs.tsx index f6c4f5c..53fe762 100644 --- a/frontend/web/src/pages/ModelConfigs.tsx +++ b/frontend/web/src/pages/ModelConfigs.tsx @@ -6,7 +6,7 @@ import { ApiOutlined, CheckCircleOutlined, DeleteOutlined, EditOutlined, PlusOutlined, ReloadOutlined, } from '@ant-design/icons' import { - modelConfigsApi, type ModelCapability, type ModelConfig, type ModelConfigPayload, + modelConfigsApi, type ModelCapability, type ModelConfig, type ModelConfigPayload, type ModelProtocol, } from '../api' import PageWrapper from '../components/PageWrapper' import { formatDateTime } from '../utils/date' @@ -24,6 +24,33 @@ const capabilityColors: Record = { moderation: 'orange', } +const protocolOptions: Record = { + openai_compatible: { + label: 'OpenAI 兼容', + capabilities: ['chat', 'embedding', 'moderation'], + endpointPlaceholder: 'https://api.example.com/v1/chat/completions', + }, + anthropic: { + label: 'Anthropic Messages', + capabilities: ['chat'], + endpointPlaceholder: 'https://api.anthropic.com/v1/messages', + }, + google_gemini: { + label: 'Google Gemini', + capabilities: ['chat'], + endpointPlaceholder: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent', + }, + dashscope: { + label: 'DashScope 原生', + capabilities: ['chat'], + endpointPlaceholder: 'https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation', + }, +} + export default function ModelConfigsPage() { const [configs, setConfigs] = useState([]) const [loading, setLoading] = useState(false) @@ -33,6 +60,8 @@ export default function ModelConfigsPage() { const [capability, setCapability] = useState() const [enabled, setEnabled] = useState() const [form] = Form.useForm() + const selectedProtocol = Form.useWatch('provider', form) || 'openai_compatible' + const selectedProtocolMeta = protocolOptions[selectedProtocol] const load = async () => { setLoading(true) @@ -72,6 +101,13 @@ export default function ModelConfigsPage() { setDrawerOpen(true) } + const changeProtocol = (provider: ModelProtocol) => { + const supportedCapabilities = protocolOptions[provider].capabilities + if (!supportedCapabilities.includes(form.getFieldValue('capability'))) { + form.setFieldValue('capability', supportedCapabilities[0]) + } + } + const submit = async () => { const values = await form.validateFields() const payload: ModelConfigPayload = { @@ -120,6 +156,10 @@ export default function ModelConfigsPage() { ), }, + { + title: '协议', dataIndex: 'provider', key: 'provider', width: 150, + render: (value: ModelProtocol) => {protocolOptions[value]?.label || value}, + }, { title: '能力', dataIndex: 'capability', key: 'capability', width: 110, render: (value: ModelCapability) => {capabilityLabels[value]}, @@ -219,14 +259,19 @@ export default function ModelConfigsPage() {
- ({ value, label: option.label }))} + /> - ({ value, label: capabilityLabels[value] }))} + />
- + {() => form.getFieldValue('capability') !== 'moderation' && ( diff --git a/tests/integration/test_model_configs_api.py b/tests/integration/test_model_configs_api.py index 080ae62..4b94264 100644 --- a/tests/integration/test_model_configs_api.py +++ b/tests/integration/test_model_configs_api.py @@ -126,3 +126,35 @@ def test_scenario_rejects_capability_mismatch(model_client): ) assert response.status_code == 400 assert "不能用于 chat" in response.json()["detail"] + + +def test_model_config_api_supports_mainstream_protocols_and_validates_capability(model_client): + client, _ = model_client + + for provider in ("anthropic", "google_gemini", "dashscope"): + response = client.post( + "/api/model-configs", + json=_payload( + name=f"{provider}-chat", + provider=provider, + endpoint_url=f"https://models.example.com/{provider}", + api_key=None, + is_default=False, + ), + ) + assert response.status_code == 200 + assert response.json()["provider"] == provider + + mismatch = client.post( + "/api/model-configs", + json=_payload( + name="anthropic-embedding", + provider="anthropic", + capability="embedding", + endpoint_url="https://models.example.com/anthropic/embeddings", + api_key=None, + is_default=False, + ), + ) + assert mismatch.status_code == 400 + assert "不支持 embedding 能力" in mismatch.json()["detail"] diff --git a/tests/unit/test_model_configs.py b/tests/unit/test_model_configs.py index f496919..51a6aec 100644 --- a/tests/unit/test_model_configs.py +++ b/tests/unit/test_model_configs.py @@ -108,3 +108,33 @@ def test_failed_binding_does_not_create_scenario(db_session): with pytest.raises(ModelConfigError): ScenarioRepository(db_session).create(scenario) assert ScenarioRepository(db_session).get(scenario.id) is None + + +@pytest.mark.parametrize("provider", ["anthropic", "google_gemini", "dashscope"]) +def test_chat_only_protocols_accept_chat_and_reject_other_capabilities(db_session, provider): + service = _service(db_session) + created = service.create( + name=f"{provider}-chat", + provider=provider, + capability="chat", + endpoint_url=f"https://models.example.com/{provider}", + model_name="chat-model", + api_key=None, + enabled=True, + is_default=False, + description="", + ) + assert created.provider == provider + + with pytest.raises(ModelConfigError, match="不支持 embedding 能力"): + service.create( + name=f"{provider}-embedding", + provider=provider, + capability="embedding", + endpoint_url=f"https://models.example.com/{provider}/embeddings", + model_name="embedding-model", + api_key=None, + enabled=True, + is_default=False, + description="", + ) diff --git a/tests/unit/test_model_gateway.py b/tests/unit/test_model_gateway.py new file mode 100644 index 0000000..c72921b --- /dev/null +++ b/tests/unit/test_model_gateway.py @@ -0,0 +1,146 @@ +"""Request and response contract tests for supported model protocols.""" + +import json +from datetime import datetime + +import httpx +import pytest +from agenteval.model_gateway import ModelGateway, ModelGatewayError +from agenteval.models import ModelCapability +from agenteval.services.model_configs import ModelRuntimeConfig + + +def _config(provider: str, capability: ModelCapability = ModelCapability.CHAT) -> ModelRuntimeConfig: + return ModelRuntimeConfig( + id=f"{provider}-config", + name=provider, + provider=provider, + capability=capability, + endpoint_url=f"https://models.example.com/{provider}", + model_name="test-model", + api_key="test-key", + updated_at=datetime(2026, 7, 17), + ) + + +def _gateway(handler) -> ModelGateway: + return ModelGateway(transport=httpx.MockTransport(handler)) + + +async def test_openai_compatible_chat_embedding_and_moderation_contracts(): + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + payload = json.loads(request.content) + if "messages" in payload: + return httpx.Response(200, json={"choices": [{"message": {"content": "chat reply"}}]}) + if payload.get("input") == ["first", "second"]: + return httpx.Response( + 200, + json={"data": [{"index": 1, "embedding": [2.0]}, {"index": 0, "embedding": [1.0]}]}, + ) + return httpx.Response(200, json={"results": [{"flagged": False}]}) + + gateway = _gateway(handler) + config = _config("openai_compatible") + + assert await gateway.chat(config, [{"role": "user", "content": "hello"}], temperature=0.3) == "chat reply" + assert await gateway.embed(config, ["first", "second"]) == [[1.0], [2.0]] + assert await gateway.moderate(config, "safe text") == {"flagged": False} + + assert all(request.headers["authorization"] == "Bearer test-key" for request in requests) + assert json.loads(requests[0].content) == { + "model": "test-model", + "messages": [{"role": "user", "content": "hello"}], + "temperature": 0.3, + } + + +async def test_anthropic_messages_contract(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.headers["x-api-key"] == "test-key" + assert request.headers["anthropic-version"] == "2023-06-01" + assert "authorization" not in request.headers + assert json.loads(request.content) == { + "model": "test-model", + "max_tokens": 1024, + "messages": [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi"}, + ], + "temperature": 0.1, + "system": "be concise", + } + return httpx.Response( + 200, + json={"content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, + ) + + result = await _gateway(handler).chat( + _config("anthropic"), + [ + {"role": "system", "content": "be concise"}, + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi"}, + ], + temperature=0.1, + ) + + assert result == "first\nsecond" + + +async def test_google_gemini_generate_content_contract(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.headers["x-goog-api-key"] == "test-key" + assert json.loads(request.content) == { + "contents": [ + {"role": "user", "parts": [{"text": "hello"}]}, + {"role": "model", "parts": [{"text": "hi"}]}, + ], + "generationConfig": {"temperature": 0.2}, + "systemInstruction": {"parts": [{"text": "be concise"}]}, + } + return httpx.Response( + 200, + json={"candidates": [{"content": {"parts": [{"text": "Gemini reply"}]}}]}, + ) + + result = await _gateway(handler).chat( + _config("google_gemini"), + [ + {"role": "system", "content": "be concise"}, + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi"}, + ], + ) + + assert result == "Gemini reply" + + +async def test_dashscope_native_contract(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.headers["authorization"] == "Bearer test-key" + assert json.loads(request.content) == { + "model": "test-model", + "input": {"messages": [{"role": "user", "content": "hello"}]}, + "parameters": {"result_format": "message", "temperature": 0.4}, + } + return httpx.Response( + 200, + json={"output": {"choices": [{"message": {"content": "DashScope reply"}}]}}, + ) + + result = await _gateway(handler).chat( + _config("dashscope"), + [{"role": "user", "content": "hello"}], + temperature=0.4, + ) + + assert result == "DashScope reply" + + +@pytest.mark.parametrize("provider", ["anthropic", "google_gemini", "dashscope"]) +async def test_chat_only_protocols_reject_embedding(provider: str): + with pytest.raises(ModelGatewayError, match="不支持 embedding 能力"): + await _gateway(lambda _: httpx.Response(500)).embed(_config(provider), "hello")