c956da7686
All checks were successful
CI / test (pull_request) Successful in 4m3s
- token 用量接入：ModelGateway 经 adapter.parse_usage 累计评测侧 LLM 调用的 token 消耗，引擎写入 run.summary.eval_token_usage，报告透出 - 放弃率落地：CaseOutcome 新增 abandoned 标记（对话中途发送/接收失败）， build_run_summary 统计 abandoned_cases / abandonment_rate - Go/No-Go 可配置：Scenario 新增 acceptance_criteria 字段（DB 列 + 幂等迁移）， 报告按场景标准出 verdict，缺省回退全局默认；标准变更不触发考纲升版
65 lines
2.2 KiB
Python
65 lines
2.2 KiB
Python
"""Verify centralized model bindings are consumed by the evaluation engine."""
|
|
|
|
from agenteval.evaluation.engine import EvalEngine
|
|
from agenteval.models import Case, CaseType, ChannelType, EvalTarget, ModelPurpose, Scenario
|
|
from agenteval.services.model_configs import ModelConfigService, SecretCipher
|
|
|
|
from tests.unit.mock_channel import MockChannel
|
|
|
|
|
|
class FakeGateway:
|
|
def __init__(self):
|
|
self.chat_calls = 0
|
|
self.total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
|
|
|
async def chat(self, config, messages, temperature=0.2):
|
|
self.chat_calls += 1
|
|
assert config.name == "生成模型"
|
|
return '["问题一", "问题二"]'
|
|
|
|
|
|
async def test_dynamic_case_uses_generator_binding_and_records_snapshot(db_session):
|
|
config = ModelConfigService(db_session, SecretCipher("")).create(
|
|
name="生成模型",
|
|
provider="openai_compatible",
|
|
capability="chat",
|
|
endpoint_url="https://models.example.com/v1/chat/completions",
|
|
model_name="generator-v1",
|
|
api_key=None,
|
|
enabled=True,
|
|
is_default=False,
|
|
description="",
|
|
)
|
|
config_id = config.id
|
|
scenario = Scenario(
|
|
id="scenario-runtime-model",
|
|
name="动态场景",
|
|
cases=[Case(id="dynamic-1", type=CaseType.DYNAMIC, prompt="生成问题", turns=2)],
|
|
model_bindings={ModelPurpose.GENERATOR: config_id},
|
|
)
|
|
target = EvalTarget(
|
|
id="target-runtime-model",
|
|
name="target",
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={
|
|
"base_url": "http://mock",
|
|
"token": "x",
|
|
"tenant": "t",
|
|
"chat_channel_id": "c",
|
|
"chat_contact_id": "u",
|
|
},
|
|
)
|
|
engine = EvalEngine(target, scenario, session=db_session)
|
|
engine.channel = MockChannel()
|
|
gateway = FakeGateway()
|
|
engine.model_gateway = gateway
|
|
|
|
run = await engine.run()
|
|
|
|
assert gateway.chat_calls == 1
|
|
assert engine.channel.sent == ["问题一", "问题二"]
|
|
snapshot = run.summary.model_configs["generator"]
|
|
assert snapshot["id"] == config_id
|
|
assert snapshot["model_name"] == "generator-v1"
|
|
assert "api_key" not in snapshot
|