782916a283
Give EvalRun.summary a typed RunSummary value (unified RunError, lenient legacy parsing) so readers stop reaching into a schemaless dict, and route every cross-run rollup — dashboard, scenario ranking, trend, campaign report — through one aggregate_runs seam. Fixes the divergence where stats averaged pass_rate over completed-only runs while the campaign report counted faults as 0.0. Cross-run rule (ADR-0004): genuine faults count 0.0, user-cancelled runs are excluded from both denominators.
64 lines
2.1 KiB
Python
64 lines
2.1 KiB
Python
"""Verify centralized model bindings are consumed by the evaluation engine."""
|
|
|
|
from agenteval.evaluation.engine import EvalEngine
|
|
from agenteval.models import Case, CaseType, ChannelType, EvalTarget, ModelPurpose, Scenario
|
|
from agenteval.services.model_configs import ModelConfigService, SecretCipher
|
|
|
|
from tests.unit.mock_channel import MockChannel
|
|
|
|
|
|
class FakeGateway:
|
|
def __init__(self):
|
|
self.chat_calls = 0
|
|
|
|
async def chat(self, config, messages, temperature=0.2):
|
|
self.chat_calls += 1
|
|
assert config.name == "生成模型"
|
|
return '["问题一", "问题二"]'
|
|
|
|
|
|
async def test_dynamic_case_uses_generator_binding_and_records_snapshot(db_session):
|
|
config = ModelConfigService(db_session, SecretCipher("")).create(
|
|
name="生成模型",
|
|
provider="openai_compatible",
|
|
capability="chat",
|
|
endpoint_url="https://models.example.com/v1/chat/completions",
|
|
model_name="generator-v1",
|
|
api_key=None,
|
|
enabled=True,
|
|
is_default=False,
|
|
description="",
|
|
)
|
|
config_id = config.id
|
|
scenario = Scenario(
|
|
id="scenario-runtime-model",
|
|
name="动态场景",
|
|
cases=[Case(id="dynamic-1", type=CaseType.DYNAMIC, prompt="生成问题", turns=2)],
|
|
model_bindings={ModelPurpose.GENERATOR: config_id},
|
|
)
|
|
target = EvalTarget(
|
|
id="target-runtime-model",
|
|
name="target",
|
|
channel_type=ChannelType.TUTU_API,
|
|
channel_config={
|
|
"base_url": "http://mock",
|
|
"token": "x",
|
|
"tenant": "t",
|
|
"chat_channel_id": "c",
|
|
"chat_contact_id": "u",
|
|
},
|
|
)
|
|
engine = EvalEngine(target, scenario, session=db_session)
|
|
engine.channel = MockChannel()
|
|
gateway = FakeGateway()
|
|
engine.model_gateway = gateway
|
|
|
|
run = await engine.run()
|
|
|
|
assert gateway.chat_calls == 1
|
|
assert engine.channel.sent == ["问题一", "问题二"]
|
|
snapshot = run.summary.model_configs["generator"]
|
|
assert snapshot["id"] == config_id
|
|
assert snapshot["model_name"] == "generator-v1"
|
|
assert "api_key" not in snapshot
|