152 lines
5.1 KiB
Python
152 lines
5.1 KiB
Python
"""Integration tests for the runner's auto analysis hook (v0.7 ticket 05).
|
||
|
||
Only 正式线 campaigns (time_scale == 1) enqueue the analysis task on
|
||
COMPLETED; accelerated/cancelled campaigns and missing analysis models all
|
||
skip silently. The analysis service itself is spied, not executed.
|
||
"""
|
||
|
||
import pytest
|
||
from agenteval.evaluation import campaign_runner
|
||
from agenteval.evaluation.campaign_lifecycle import cancel_campaign
|
||
from agenteval.evaluation.campaign_runner import CampaignRuntime
|
||
from agenteval.models import (
|
||
Campaign,
|
||
CampaignPlanEntry,
|
||
CampaignStatus,
|
||
Case,
|
||
CaseType,
|
||
ChannelType,
|
||
EvalTarget,
|
||
PlatformType,
|
||
Scenario,
|
||
TargetStatus,
|
||
)
|
||
from agenteval.storage.repository import CampaignRepository, ScenarioRepository, TargetRepository
|
||
|
||
from tests.unit.mock_channel import MockChannel
|
||
|
||
TICK = 0.01
|
||
|
||
|
||
@pytest.fixture()
|
||
def seeded_db(db_session, monkeypatch):
|
||
from agenteval.channels import factory as factory_module
|
||
from agenteval.evaluation import engine as engine_module
|
||
from agenteval.storage import db as db_module
|
||
from agenteval.storage import repository as repo_module
|
||
|
||
def _test_get_session():
|
||
return db_session
|
||
|
||
monkeypatch.setattr(db_module, "get_session", _test_get_session)
|
||
monkeypatch.setattr(repo_module, "get_session", _test_get_session)
|
||
monkeypatch.setattr(engine_module, "get_session", _test_get_session)
|
||
monkeypatch.setattr(campaign_runner, "get_session", _test_get_session)
|
||
|
||
channel = MockChannel(reply_delay=0.0)
|
||
monkeypatch.setattr(factory_module.ChannelFactory, "create", lambda target: channel)
|
||
|
||
TargetRepository(db_session).create(
|
||
EvalTarget(
|
||
id="t-1",
|
||
name="mock-target",
|
||
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
|
||
channel_type=ChannelType.TUTU_API,
|
||
channel_config={"base_url": "http://mock", "token": "x"},
|
||
status=TargetStatus.ACTIVE,
|
||
)
|
||
)
|
||
ScenarioRepository(db_session).create(
|
||
Scenario(
|
||
id="s-1",
|
||
name="mock-scenario",
|
||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||
)
|
||
)
|
||
return db_session
|
||
|
||
|
||
@pytest.fixture()
|
||
def analysis_spy(monkeypatch):
|
||
"""Spy the analysis seam: resolvable model, recorded enqueue calls."""
|
||
calls: list[tuple[str, str]] = []
|
||
monkeypatch.setattr(
|
||
campaign_runner,
|
||
"enqueue_campaign_analysis",
|
||
lambda cid, *, triggered_by: calls.append((cid, triggered_by)),
|
||
)
|
||
monkeypatch.setattr(campaign_runner, "resolve_analysis_model", lambda campaign, session: object())
|
||
return calls
|
||
|
||
|
||
def _make_campaign(session, **overrides) -> Campaign:
|
||
payload = dict(
|
||
name="auto-trigger",
|
||
target_id="t-1",
|
||
window_seconds=1,
|
||
time_scale=1.0, # 正式线：1s 窗口真实耗时 ~1s
|
||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
||
)
|
||
payload.update(overrides)
|
||
return CampaignRepository(session).create(Campaign(**payload))
|
||
|
||
|
||
@pytest.fixture()
|
||
async def runtime(seeded_db):
|
||
value = CampaignRuntime(session_factory=lambda: seeded_db, tick_seconds=TICK)
|
||
yield value
|
||
await value.shutdown()
|
||
|
||
|
||
async def _await_terminal(session, campaign_id, timeout=5.0):
|
||
import asyncio
|
||
|
||
async with asyncio.timeout(timeout):
|
||
while CampaignRepository(session).get(campaign_id).status is CampaignStatus.RUNNING:
|
||
await asyncio.sleep(TICK)
|
||
|
||
|
||
async def test_realtime_completion_auto_enqueues_analysis(seeded_db, runtime, analysis_spy):
|
||
campaign = _make_campaign(seeded_db)
|
||
assert runtime.start(campaign.id)
|
||
await _await_terminal(seeded_db, campaign.id)
|
||
|
||
final = CampaignRepository(seeded_db).get(campaign.id)
|
||
assert final.status == CampaignStatus.COMPLETED
|
||
assert analysis_spy == [(campaign.id, "auto")]
|
||
|
||
|
||
async def test_accelerated_completion_does_not_enqueue(seeded_db, runtime, analysis_spy):
|
||
campaign = _make_campaign(seeded_db, time_scale=1000.0) # 加速调试线
|
||
assert runtime.start(campaign.id)
|
||
await _await_terminal(seeded_db, campaign.id)
|
||
|
||
final = CampaignRepository(seeded_db).get(campaign.id)
|
||
assert final.status == CampaignStatus.COMPLETED
|
||
assert analysis_spy == []
|
||
|
||
|
||
async def test_cancelled_campaign_does_not_enqueue(seeded_db, runtime, analysis_spy):
|
||
import asyncio
|
||
|
||
campaign = _make_campaign(seeded_db, window_seconds=100)
|
||
assert runtime.start(campaign.id)
|
||
await asyncio.sleep(0.05)
|
||
|
||
repo = CampaignRepository(seeded_db)
|
||
cancel_campaign(seeded_db, campaign.id, stop=runtime.cancel)
|
||
|
||
assert repo.get(campaign.id).status == CampaignStatus.CANCELLED
|
||
assert analysis_spy == []
|
||
|
||
|
||
async def test_missing_analysis_model_skips_silently(seeded_db, runtime, monkeypatch, analysis_spy):
|
||
monkeypatch.setattr(campaign_runner, "resolve_analysis_model", lambda campaign, session: None)
|
||
campaign = _make_campaign(seeded_db)
|
||
assert runtime.start(campaign.id)
|
||
await _await_terminal(seeded_db, campaign.id)
|
||
|
||
final = CampaignRepository(seeded_db).get(campaign.id)
|
||
assert final.status == CampaignStatus.COMPLETED # 活动完成流程不受影响
|
||
assert analysis_spy == []
|