2f09ee2bfc
All checks were successful
CI / test (pull_request) Successful in 3m58s
- 报告渲染 Go/No-Go 上线评估横幅（HTML 彩色 banner + Markdown 引用块） - 抽取 scored_llm 共享模块：llm_score / fluency 直连调用与评分解析收敛 - 网关新增 chat_with_usage / embed_with_usage，规则按次归集 llm_usage - 引擎分岗位用量归集（judge/generator/embedding/moderation）写入 RunSummary.eval_usage_by_purpose，并发下不做总量差值 - cost_tracking 重构：data/model_pricing.json 覆盖 + 默认计价表， 删除从未有数据支撑的 Turn 维度成本函数（偏差说明见 PR） - 报告 summary 增加 eval_cost 分岗位成本段并在 Markdown 渲染
71 lines
2.6 KiB
Python
71 lines
2.6 KiB
Python
"""Verify centralized model bindings are consumed by the evaluation engine."""
|
||
|
||
from agenteval.evaluation.engine import EvalEngine
|
||
from agenteval.models import Case, CaseType, ChannelType, EvalTarget, ModelPurpose, Scenario
|
||
from agenteval.services.model_configs import ModelConfigService, SecretCipher
|
||
|
||
from tests.unit.mock_channel import MockChannel
|
||
|
||
|
||
class FakeGateway:
|
||
def __init__(self):
|
||
self.chat_calls = 0
|
||
self.total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
||
|
||
async def chat_with_usage(self, config, messages, temperature=0.2):
|
||
self.chat_calls += 1
|
||
assert config.name == "生成模型"
|
||
return '["问题一", "问题二"]', {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
|
||
|
||
async def chat(self, config, messages, temperature=0.2):
|
||
content, _ = await self.chat_with_usage(config, messages, temperature)
|
||
return content
|
||
|
||
|
||
async def test_dynamic_case_uses_generator_binding_and_records_snapshot(db_session):
|
||
config = ModelConfigService(db_session, SecretCipher("")).create(
|
||
name="生成模型",
|
||
provider="openai_compatible",
|
||
capability="chat",
|
||
endpoint_url="https://models.example.com/v1/chat/completions",
|
||
model_name="generator-v1",
|
||
api_key=None,
|
||
enabled=True,
|
||
is_default=False,
|
||
description="",
|
||
)
|
||
config_id = config.id
|
||
scenario = Scenario(
|
||
id="scenario-runtime-model",
|
||
name="动态场景",
|
||
cases=[Case(id="dynamic-1", type=CaseType.DYNAMIC, prompt="生成问题", turns=2)],
|
||
model_bindings={ModelPurpose.GENERATOR: config_id},
|
||
)
|
||
target = EvalTarget(
|
||
id="target-runtime-model",
|
||
name="target",
|
||
channel_type=ChannelType.TUTU_API,
|
||
channel_config={
|
||
"base_url": "http://mock",
|
||
"token": "x",
|
||
"tenant": "t",
|
||
"chat_channel_id": "c",
|
||
"chat_contact_id": "u",
|
||
},
|
||
)
|
||
engine = EvalEngine(target, scenario, session=db_session)
|
||
engine.channel = MockChannel()
|
||
gateway = FakeGateway()
|
||
engine.model_gateway = gateway
|
||
|
||
run = await engine.run()
|
||
|
||
assert gateway.chat_calls == 1
|
||
assert engine.channel.sent == ["问题一", "问题二"]
|
||
snapshot = run.summary.model_configs["generator"]
|
||
assert snapshot["id"] == config_id
|
||
assert snapshot["model_name"] == "generator-v1"
|
||
assert "api_key" not in snapshot
|
||
# 生成岗位的用量按次归集进 summary（分岗位成本核算的数据源）
|
||
assert run.summary.eval_usage_by_purpose["generator"]["total_tokens"] == 15
|