"""Phase 2 (v1.3.1) wiring tests: token usage, abandonment rate, go/no-go config.""" from datetime import datetime import httpx import pytest from agenteval.channels.base import ChannelTransportError from agenteval.evaluation.judgement import CaseOutcome from agenteval.evaluation.run_summary import build_run_summary from agenteval.model_gateway import ModelGateway from agenteval.models import ( Case, CaseType, ChannelType, EvalTarget, ModelCapability, PlatformType, RunStatus, Scenario, TargetStatus, ) from agenteval.services.model_configs import ModelRuntimeConfig from agenteval.storage.repository import RunRepository, ScenarioRepository from tests.unit.mock_channel import MockChannel from tests.unit.test_engine import _build_engine # ── 2.1 token usage accumulation ──────────────────────────────────────── def _runtime_config() -> ModelRuntimeConfig: return ModelRuntimeConfig( id="cfg-1", name="judge", provider="openai_compatible", capability=ModelCapability.CHAT, endpoint_url="https://models.example.com/v1/chat/completions", model_name="test-model", api_key="k", updated_at=datetime(2026, 8, 25), ) @pytest.mark.asyncio async def test_gateway_accumulates_token_usage(): def handler(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, json={ "choices": [{"message": {"content": "ok"}}], "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, }, ) gateway = ModelGateway(transport=httpx.MockTransport(handler)) config = _runtime_config() await gateway.chat(config, [{"role": "user", "content": "a"}]) await gateway.chat(config, [{"role": "user", "content": "b"}]) assert gateway.total_usage == {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30} @pytest.mark.asyncio async def test_gateway_usage_stays_zero_when_response_omits_it(): def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(200, json={"choices": [{"message": {"content": "ok"}}]}) gateway = ModelGateway(transport=httpx.MockTransport(handler)) await gateway.chat(_runtime_config(), [{"role": "user", "content": "a"}]) assert gateway.total_usage["total_tokens"] == 0 # ── 2.2 abandonment rate ──────────────────────────────────────────────── def test_build_run_summary_counts_abandoned_cases(): outcomes = { "c1": CaseOutcome(passed=True, connectivity=False), "c2": CaseOutcome(passed=False, connectivity=False, abandoned=True), } summary = build_run_summary( case_outcomes=outcomes, latencies=[100], rule_passes=[True], eval_token_usage={"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, ) assert summary.abandoned_cases == 1 assert summary.abandonment_rate == 0.5 assert summary.eval_token_usage is not None assert summary.eval_token_usage["total_tokens"] == 12 assert summary.case_outcomes["c2"].abandoned is True assert summary.case_outcomes["c1"].abandoned is False class _FailSecondPoll(MockChannel): """First poll succeeds, subsequent polls raise — dialog abandoned mid-way.""" async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0): if self.poll_calls >= 1: raise ChannelTransportError("upstream gone") return await super()._poll_reply(question_msg_id, timeout, poll_interval) def _two_message_scenario() -> Scenario: return Scenario( id="s-1", name="abandon", cases=[Case(id="c1", type=CaseType.SINGLE, messages=["m1", "m2"])], ) def _target() -> EvalTarget: return EvalTarget( id="t-1", name="mock-target", platform=PlatformType.AI_DIGITAL_EMPLOYEE, channel_type=ChannelType.TUTU_API, channel_config={ "base_url": "http://mock", "token": "x", "tenant": "t", "chat_channel_id": "c", "chat_contact_id": "u", }, status=TargetStatus.ACTIVE, ) @pytest.mark.asyncio async def test_engine_marks_mid_dialog_failure_as_abandoned(db_session): engine = _build_engine(_two_message_scenario(), _FailSecondPoll(), session=db_session) run = await engine.run() assert run.status == RunStatus.COMPLETED assert run.summary.abandoned_cases == 1 assert run.summary.abandonment_rate == 1.0 assert run.summary.case_outcomes["c1"].abandoned is True @pytest.mark.asyncio async def test_engine_first_round_failure_is_not_abandoned(db_session): # 第一轮发送就失败：连通性问题，不算放弃 engine = _build_engine( _two_message_scenario(), MockChannel(send_ok=False), session=db_session ) run = await engine.run() assert run.summary.abandoned_cases == 0 assert run.summary.case_outcomes["c1"].abandoned is False # ── 2.3 go/no-go per-scenario acceptance criteria ─────────────────────── def test_scenario_acceptance_criteria_roundtrip(db_session): repo = ScenarioRepository(db_session) criteria = {"judged_pass_rate_min": 0.8, "avg_latency_max_ms": 5000} scenario = Scenario( name="criteria", cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])], acceptance_criteria=criteria, ) created = repo.create(scenario) fetched = repo.get(created.id) assert fetched is not None assert fetched.acceptance_criteria == criteria # 验收标准是报告配置，不属于考纲——变更不应升版 fetched.acceptance_criteria = {"judged_pass_rate_min": 0.7} updated = repo.update(fetched) assert updated is not None assert updated.version == created.version assert updated.acceptance_criteria == {"judged_pass_rate_min": 0.7} @pytest.mark.asyncio async def test_report_verdict_uses_scenario_criteria(report_seeded): from agenteval.evaluation.report import generate_report session, run_id, scenario_id = report_seeded # 默认标准（judged ≥ 0.95）下该 run 为 no_go report = generate_report(run_id, session) assert report["go_no_go"]["decision"] == "no_go" # 场景放宽标准后翻转为 go repo = ScenarioRepository(session) scenario = repo.get(scenario_id) assert scenario is not None scenario.acceptance_criteria = {"judged_pass_rate_min": 0.0, "pass_rate_min": 0.0} repo.update(scenario) report = generate_report(run_id, session) assert report["go_no_go"]["decision"] == "go" @pytest.fixture() def report_seeded(db_session): """Seed a completed failing run (pass_rate 0) and return (session, run_id, scenario_id).""" from agenteval.models import EvalResult, EvalRun, Turn from agenteval.storage.repository import ResultRepository, TargetRepository target = TargetRepository(db_session).create( EvalTarget( name="t", platform=PlatformType.AI_DIGITAL_EMPLOYEE, channel_type=ChannelType.TUTU_API, channel_config={}, status=TargetStatus.ACTIVE, ) ) scenario = ScenarioRepository(db_session).create( Scenario(name="s", cases=[Case(id="c0", type=CaseType.SINGLE, messages=["hi"])]) ) run = RunRepository(db_session).create( EvalRun( target_id=target.id, scenario_id=scenario.id, scenario_version=1, status=RunStatus.COMPLETED, ) ) result_repo = ResultRepository(db_session) result_repo.save_turn( Turn( run_id=run.id, case_id="c0", round_index=1, sent_message={"msgBody": {"content": "hi"}}, reply={"msgBody": {"content": "bad answer"}}, latency_ms=200, ) ) db_turn = RunRepository(db_session).get_turns(run.id)[-1] result_repo.save_result( EvalResult( run_id=run.id, case_id="c0", turn_id=db_turn.id or "", rule_type="keyword_match", passed=False, score=0.0, reason="失败", ) ) run.summary = { "total_cases": 1, "passed_cases": 0, "failed_cases": 1, "total_rules": 1, "passed_rules": 0, "pass_rate": 0.0, "judged_pass_rate": 0.0, "avg_latency_ms": 200.0, "case_outcomes": {"c0": {"passed": False, "connectivity": False}}, } RunRepository(db_session).update(run) return db_session, run.id, scenario.id