e1e067bac4
Introduce ModelPurpose.ANALYSIS and a globally-unique is_analysis_default marker on chat model configs so campaign analysis can resolve its model. Service rejects disabled or non-chat configs; repo clears the previous holder on set. Documented the analysis role in CONTEXT.md.
239 lines
8.1 KiB
Python
239 lines
8.1 KiB
Python
"""Integration tests for the centralized model configuration API."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from agenteval.config import get_settings
|
|
from agenteval.storage.db import ModelConfigDB
|
|
from agenteval.web.app import app
|
|
from agenteval.web.deps import get_db
|
|
from cryptography.fernet import Fernet
|
|
from fastapi.testclient import TestClient
|
|
from sqlmodel import Session, SQLModel, create_engine
|
|
|
|
|
|
@pytest.fixture()
|
|
def model_client(tmp_path: Path, monkeypatch):
|
|
engine = create_engine(
|
|
f"sqlite:///{tmp_path / 'models_api.db'}",
|
|
connect_args={"check_same_thread": False},
|
|
)
|
|
SQLModel.metadata.create_all(engine)
|
|
session = Session(engine)
|
|
|
|
from agenteval.web import app as app_module
|
|
|
|
monkeypatch.setattr(app_module, "init_db", lambda: None)
|
|
monkeypatch.setattr(get_settings(), "secret_key", Fernet.generate_key().decode("ascii"))
|
|
|
|
def override_get_db():
|
|
yield session
|
|
|
|
app.dependency_overrides[get_db] = override_get_db
|
|
with TestClient(app) as client:
|
|
yield client, session
|
|
|
|
app.dependency_overrides.clear()
|
|
session.close()
|
|
engine.dispose()
|
|
|
|
|
|
def _payload(**overrides) -> dict:
|
|
payload = {
|
|
"name": "评估对话模型",
|
|
"provider": "openai_compatible",
|
|
"capability": "chat",
|
|
"endpoint_url": "https://models.example.com/v1/chat/completions",
|
|
"model_name": "judge-model",
|
|
"api_key": "sk-private",
|
|
"enabled": True,
|
|
"is_default": True,
|
|
"description": "integration test",
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
def test_model_config_crud_never_returns_secret(model_client):
|
|
client, session = model_client
|
|
|
|
metadata = {
|
|
"vendor_name": "Example AI",
|
|
"input_modalities": ["text", "image"],
|
|
"output_modalities": ["text"],
|
|
"context_window": 128000,
|
|
"max_output_tokens": 8192,
|
|
"supports_streaming": True,
|
|
"supports_tool_calling": True,
|
|
"supports_structured_output": True,
|
|
"supports_reasoning": False,
|
|
"region": "cn-test-1",
|
|
"documentation_url": "https://models.example.com/docs",
|
|
}
|
|
created_response = client.post("/api/model-configs", json=_payload(**metadata))
|
|
assert created_response.status_code == 200
|
|
created = created_response.json()
|
|
assert created["has_api_key"] is True
|
|
assert "api_key" not in created
|
|
assert "api_key_encrypted" not in created
|
|
assert "sk-private" not in created_response.text
|
|
assert created["vendor_name"] == "Example AI"
|
|
assert created["input_modalities"] == ["text", "image"]
|
|
assert created["output_modalities"] == ["text"]
|
|
assert created["context_window"] == 128000
|
|
assert created["max_output_tokens"] == 8192
|
|
assert created["supports_tool_calling"] is True
|
|
assert created["region"] == "cn-test-1"
|
|
|
|
stored = session.get(ModelConfigDB, created["id"])
|
|
assert stored and stored.api_key_encrypted
|
|
assert stored.api_key_encrypted != "sk-private"
|
|
|
|
listed = client.get("/api/model-configs").json()
|
|
assert len(listed) == 1
|
|
assert "sk-private" not in str(listed)
|
|
|
|
updated_response = client.put(
|
|
f"/api/model-configs/{created['id']}",
|
|
json=_payload(api_key=None, clear_api_key=False, model_name="judge-model-v2", **metadata),
|
|
)
|
|
assert updated_response.status_code == 200
|
|
assert updated_response.json()["has_api_key"] is True
|
|
assert updated_response.json()["model_name"] == "judge-model-v2"
|
|
assert updated_response.json()["input_modalities"] == ["text", "image"]
|
|
|
|
assert client.delete(f"/api/model-configs/{created['id']}").status_code == 200
|
|
assert client.get(f"/api/model-configs/{created['id']}").status_code == 404
|
|
|
|
|
|
def test_referenced_config_cannot_be_deleted(model_client):
|
|
client, _ = model_client
|
|
config = client.post("/api/model-configs", json=_payload(api_key=None)).json()
|
|
scenario = {
|
|
"name": "引用模型的场景",
|
|
"cases": [{"id": "case-1", "messages": ["hello"]}],
|
|
"model_bindings": {"judge": config["id"]},
|
|
}
|
|
|
|
scenario_response = client.post("/api/scenarios", json=scenario)
|
|
assert scenario_response.status_code == 200
|
|
scenario_id = scenario_response.json()["id"]
|
|
|
|
references = client.get(f"/api/model-configs/{config['id']}/references").json()
|
|
assert references == [{"scenario_id": scenario_id, "scenario_name": "引用模型的场景", "purpose": "judge"}]
|
|
|
|
delete_response = client.delete(f"/api/model-configs/{config['id']}")
|
|
assert delete_response.status_code == 409
|
|
|
|
|
|
def test_scenario_rejects_capability_mismatch(model_client):
|
|
client, _ = model_client
|
|
embedding = client.post(
|
|
"/api/model-configs",
|
|
json=_payload(
|
|
name="向量模型",
|
|
capability="embedding",
|
|
endpoint_url="https://models.example.com/v1/embeddings",
|
|
),
|
|
).json()
|
|
|
|
response = client.post(
|
|
"/api/scenarios",
|
|
json={
|
|
"name": "错误绑定",
|
|
"cases": [{"id": "case-1", "messages": ["hello"]}],
|
|
"model_bindings": {"judge": embedding["id"]},
|
|
},
|
|
)
|
|
assert response.status_code == 400
|
|
assert "不能用于 chat" in response.json()["detail"]
|
|
|
|
|
|
def test_model_config_api_supports_mainstream_protocols_and_validates_capability(model_client):
|
|
client, _ = model_client
|
|
|
|
for provider in ("anthropic", "google_gemini", "dashscope"):
|
|
response = client.post(
|
|
"/api/model-configs",
|
|
json=_payload(
|
|
name=f"{provider}-chat",
|
|
provider=provider,
|
|
endpoint_url=f"https://models.example.com/{provider}",
|
|
api_key=None,
|
|
is_default=False,
|
|
),
|
|
)
|
|
assert response.status_code == 200
|
|
assert response.json()["provider"] == provider
|
|
|
|
mismatch = client.post(
|
|
"/api/model-configs",
|
|
json=_payload(
|
|
name="anthropic-embedding",
|
|
provider="anthropic",
|
|
capability="embedding",
|
|
endpoint_url="https://models.example.com/anthropic/embeddings",
|
|
api_key=None,
|
|
is_default=False,
|
|
),
|
|
)
|
|
assert mismatch.status_code == 400
|
|
assert "不支持 embedding 能力" in mismatch.json()["detail"]
|
|
|
|
|
|
def test_model_config_metadata_defaults_and_validation(model_client):
|
|
client, _ = model_client
|
|
created = client.post("/api/model-configs", json=_payload(api_key=None)).json()
|
|
assert created["input_modalities"] == ["text"]
|
|
assert created["output_modalities"] == ["text"]
|
|
assert created["vendor_name"] == ""
|
|
assert created["context_window"] is None
|
|
assert created["supports_reasoning"] is False
|
|
|
|
invalid_modality = client.post(
|
|
"/api/model-configs",
|
|
json=_payload(name="invalid-modality", api_key=None, input_modalities=["document"]),
|
|
)
|
|
assert invalid_modality.status_code == 422
|
|
|
|
invalid_context = client.post(
|
|
"/api/model-configs",
|
|
json=_payload(name="invalid-context", api_key=None, context_window=0),
|
|
)
|
|
assert invalid_context.status_code == 422
|
|
|
|
|
|
def test_analysis_default_via_api(model_client):
|
|
client, _ = model_client
|
|
|
|
first = client.post(
|
|
"/api/model-configs",
|
|
json=_payload(name="分析模型一", api_key=None, is_default=False, is_analysis_default=True),
|
|
)
|
|
assert first.status_code == 200
|
|
assert first.json()["is_analysis_default"] is True
|
|
|
|
second = client.post(
|
|
"/api/model-configs",
|
|
json=_payload(name="分析模型二", api_key=None, is_default=False, is_analysis_default=True),
|
|
)
|
|
assert second.status_code == 200
|
|
|
|
listed = client.get("/api/model-configs").json()
|
|
flags = {item["name"]: item["is_analysis_default"] for item in listed}
|
|
assert flags == {"分析模型一": False, "分析模型二": True}
|
|
|
|
rejected = client.post(
|
|
"/api/model-configs",
|
|
json=_payload(
|
|
name="向量分析",
|
|
capability="embedding",
|
|
model_name="embed-model",
|
|
api_key=None,
|
|
is_default=False,
|
|
is_analysis_default=True,
|
|
),
|
|
)
|
|
assert rejected.status_code == 400
|
|
assert "分析默认" in rejected.json()["detail"]
|