c956da7686
All checks were successful
CI / test (pull_request) Successful in 4m3s
- token 用量接入：ModelGateway 经 adapter.parse_usage 累计评测侧 LLM 调用的 token 消耗，引擎写入 run.summary.eval_token_usage，报告透出 - 放弃率落地：CaseOutcome 新增 abandoned 标记（对话中途发送/接收失败）， build_run_summary 统计 abandoned_cases / abandonment_rate - Go/No-Go 可配置：Scenario 新增 acceptance_criteria 字段（DB 列 + 幂等迁移）， 报告按场景标准出 verdict，缺省回退全局默认；标准变更不触发考纲升版
256 lines
8.7 KiB
Python
256 lines
8.7 KiB
Python
"""Phase 2 (v1.3.1) wiring tests: token usage, abandonment rate, go/no-go config."""
|
||
|
||
from datetime import datetime
|
||
|
||
import httpx
|
||
import pytest
|
||
from agenteval.channels.base import ChannelTransportError
|
||
from agenteval.evaluation.judgement import CaseOutcome
|
||
from agenteval.evaluation.run_summary import build_run_summary
|
||
from agenteval.model_gateway import ModelGateway
|
||
from agenteval.models import (
|
||
Case,
|
||
CaseType,
|
||
ChannelType,
|
||
EvalTarget,
|
||
ModelCapability,
|
||
PlatformType,
|
||
RunStatus,
|
||
Scenario,
|
||
TargetStatus,
|
||
)
|
||
from agenteval.services.model_configs import ModelRuntimeConfig
|
||
from agenteval.storage.repository import RunRepository, ScenarioRepository
|
||
|
||
from tests.unit.mock_channel import MockChannel
|
||
from tests.unit.test_engine import _build_engine
|
||
|
||
# ── 2.1 token usage accumulation ────────────────────────────────────────
|
||
|
||
|
||
def _runtime_config() -> ModelRuntimeConfig:
|
||
return ModelRuntimeConfig(
|
||
id="cfg-1",
|
||
name="judge",
|
||
provider="openai_compatible",
|
||
capability=ModelCapability.CHAT,
|
||
endpoint_url="https://models.example.com/v1/chat/completions",
|
||
model_name="test-model",
|
||
api_key="k",
|
||
updated_at=datetime(2026, 8, 25),
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gateway_accumulates_token_usage():
|
||
def handler(request: httpx.Request) -> httpx.Response:
|
||
return httpx.Response(
|
||
200,
|
||
json={
|
||
"choices": [{"message": {"content": "ok"}}],
|
||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||
},
|
||
)
|
||
|
||
gateway = ModelGateway(transport=httpx.MockTransport(handler))
|
||
config = _runtime_config()
|
||
await gateway.chat(config, [{"role": "user", "content": "a"}])
|
||
await gateway.chat(config, [{"role": "user", "content": "b"}])
|
||
assert gateway.total_usage == {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gateway_usage_stays_zero_when_response_omits_it():
|
||
def handler(request: httpx.Request) -> httpx.Response:
|
||
return httpx.Response(200, json={"choices": [{"message": {"content": "ok"}}]})
|
||
|
||
gateway = ModelGateway(transport=httpx.MockTransport(handler))
|
||
await gateway.chat(_runtime_config(), [{"role": "user", "content": "a"}])
|
||
assert gateway.total_usage["total_tokens"] == 0
|
||
|
||
|
||
# ── 2.2 abandonment rate ────────────────────────────────────────────────
|
||
|
||
|
||
def test_build_run_summary_counts_abandoned_cases():
|
||
outcomes = {
|
||
"c1": CaseOutcome(passed=True, connectivity=False),
|
||
"c2": CaseOutcome(passed=False, connectivity=False, abandoned=True),
|
||
}
|
||
summary = build_run_summary(
|
||
case_outcomes=outcomes,
|
||
latencies=[100],
|
||
rule_passes=[True],
|
||
eval_token_usage={"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12},
|
||
)
|
||
assert summary.abandoned_cases == 1
|
||
assert summary.abandonment_rate == 0.5
|
||
assert summary.eval_token_usage is not None
|
||
assert summary.eval_token_usage["total_tokens"] == 12
|
||
assert summary.case_outcomes["c2"].abandoned is True
|
||
assert summary.case_outcomes["c1"].abandoned is False
|
||
|
||
|
||
class _FailSecondPoll(MockChannel):
|
||
"""First poll succeeds, subsequent polls raise — dialog abandoned mid-way."""
|
||
|
||
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
|
||
if self.poll_calls >= 1:
|
||
raise ChannelTransportError("upstream gone")
|
||
return await super()._poll_reply(question_msg_id, timeout, poll_interval)
|
||
|
||
|
||
def _two_message_scenario() -> Scenario:
|
||
return Scenario(
|
||
id="s-1",
|
||
name="abandon",
|
||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["m1", "m2"])],
|
||
)
|
||
|
||
|
||
def _target() -> EvalTarget:
|
||
return EvalTarget(
|
||
id="t-1",
|
||
name="mock-target",
|
||
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
||
channel_type=ChannelType.TUTU_API,
|
||
channel_config={
|
||
"base_url": "http://mock",
|
||
"token": "x",
|
||
"tenant": "t",
|
||
"chat_channel_id": "c",
|
||
"chat_contact_id": "u",
|
||
},
|
||
status=TargetStatus.ACTIVE,
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_engine_marks_mid_dialog_failure_as_abandoned(db_session):
|
||
engine = _build_engine(_two_message_scenario(), _FailSecondPoll(), session=db_session)
|
||
run = await engine.run()
|
||
assert run.status == RunStatus.COMPLETED
|
||
assert run.summary.abandoned_cases == 1
|
||
assert run.summary.abandonment_rate == 1.0
|
||
assert run.summary.case_outcomes["c1"].abandoned is True
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_engine_first_round_failure_is_not_abandoned(db_session):
|
||
# 第一轮发送就失败：连通性问题，不算放弃
|
||
engine = _build_engine(
|
||
_two_message_scenario(), MockChannel(send_ok=False), session=db_session
|
||
)
|
||
run = await engine.run()
|
||
assert run.summary.abandoned_cases == 0
|
||
assert run.summary.case_outcomes["c1"].abandoned is False
|
||
|
||
|
||
# ── 2.3 go/no-go per-scenario acceptance criteria ───────────────────────
|
||
|
||
|
||
def test_scenario_acceptance_criteria_roundtrip(db_session):
|
||
repo = ScenarioRepository(db_session)
|
||
criteria = {"judged_pass_rate_min": 0.8, "avg_latency_max_ms": 5000}
|
||
scenario = Scenario(
|
||
name="criteria",
|
||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||
acceptance_criteria=criteria,
|
||
)
|
||
created = repo.create(scenario)
|
||
fetched = repo.get(created.id)
|
||
assert fetched is not None
|
||
assert fetched.acceptance_criteria == criteria
|
||
|
||
# 验收标准是报告配置，不属于考纲——变更不应升版
|
||
fetched.acceptance_criteria = {"judged_pass_rate_min": 0.7}
|
||
updated = repo.update(fetched)
|
||
assert updated is not None
|
||
assert updated.version == created.version
|
||
assert updated.acceptance_criteria == {"judged_pass_rate_min": 0.7}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_report_verdict_uses_scenario_criteria(report_seeded):
|
||
from agenteval.evaluation.report import generate_report
|
||
|
||
session, run_id, scenario_id = report_seeded
|
||
|
||
# 默认标准（judged ≥ 0.95）下该 run 为 no_go
|
||
report = generate_report(run_id, session)
|
||
assert report["go_no_go"]["decision"] == "no_go"
|
||
|
||
# 场景放宽标准后翻转为 go
|
||
repo = ScenarioRepository(session)
|
||
scenario = repo.get(scenario_id)
|
||
assert scenario is not None
|
||
scenario.acceptance_criteria = {"judged_pass_rate_min": 0.0, "pass_rate_min": 0.0}
|
||
repo.update(scenario)
|
||
|
||
report = generate_report(run_id, session)
|
||
assert report["go_no_go"]["decision"] == "go"
|
||
|
||
|
||
@pytest.fixture()
|
||
def report_seeded(db_session):
|
||
"""Seed a completed failing run (pass_rate 0) and return (session, run_id, scenario_id)."""
|
||
from agenteval.models import EvalResult, EvalRun, Turn
|
||
from agenteval.storage.repository import ResultRepository, TargetRepository
|
||
|
||
target = TargetRepository(db_session).create(
|
||
EvalTarget(
|
||
name="t",
|
||
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
||
channel_type=ChannelType.TUTU_API,
|
||
channel_config={},
|
||
status=TargetStatus.ACTIVE,
|
||
)
|
||
)
|
||
scenario = ScenarioRepository(db_session).create(
|
||
Scenario(name="s", cases=[Case(id="c0", type=CaseType.SINGLE, messages=["hi"])])
|
||
)
|
||
run = RunRepository(db_session).create(
|
||
EvalRun(
|
||
target_id=target.id,
|
||
scenario_id=scenario.id,
|
||
scenario_version=1,
|
||
status=RunStatus.COMPLETED,
|
||
)
|
||
)
|
||
result_repo = ResultRepository(db_session)
|
||
result_repo.save_turn(
|
||
Turn(
|
||
run_id=run.id,
|
||
case_id="c0",
|
||
round_index=1,
|
||
sent_message={"msgBody": {"content": "hi"}},
|
||
reply={"msgBody": {"content": "bad answer"}},
|
||
latency_ms=200,
|
||
)
|
||
)
|
||
db_turn = RunRepository(db_session).get_turns(run.id)[-1]
|
||
result_repo.save_result(
|
||
EvalResult(
|
||
run_id=run.id,
|
||
case_id="c0",
|
||
turn_id=db_turn.id or "",
|
||
rule_type="keyword_match",
|
||
passed=False,
|
||
score=0.0,
|
||
reason="失败",
|
||
)
|
||
)
|
||
run.summary = {
|
||
"total_cases": 1,
|
||
"passed_cases": 0,
|
||
"failed_cases": 1,
|
||
"total_rules": 1,
|
||
"passed_rules": 0,
|
||
"pass_rate": 0.0,
|
||
"judged_pass_rate": 0.0,
|
||
"avg_latency_ms": 200.0,
|
||
"case_outcomes": {"c0": {"passed": False, "connectivity": False}},
|
||
}
|
||
RunRepository(db_session).update(run)
|
||
return db_session, run.id, scenario.id
|