AgentEvalTool/tests/integration/test_campaign_comparison_auto_trigger.py

"""Auto-chain tests: analysis completed → period comparison enqueued (v0.8 ticket 04).

Only 正式线 campaigns with a resolvable model and an auto baseline enqueue the
comparison task; all other branches skip silently without touching the analysis
row. The comparison entry point is spied, not executed.
"""

import json
from datetime import timedelta
from types import SimpleNamespace

import pytest
from agenteval.evaluation import analysis as analysis_module
from agenteval.evaluation import comparison as comparison_module
from agenteval.evaluation.analysis import execute_campaign_analysis
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus, RunSummary
from agenteval.storage.db import CampaignAnalysisDB, utc_now
from agenteval.storage.model_config_repository import ModelConfigRepository
from agenteval.storage.repository import CampaignRepository, RunRepository
from sqlmodel import select

T0 = utc_now().replace(tzinfo=None) - timedelta(hours=3)

STAGE1 = json.dumps({"narrative": "售前场景表现稳定", "problems": []})
STAGE2 = json.dumps({"overall": "整体达标", "problems": [], "suggestions": []})


class FakeChatClient:
    def __init__(self, *responses):
        self._responses = list(responses)

    async def __call__(self, messages: list[dict]) -> str:
        return self._responses.pop(0)


def _campaign(campaign_id: str, *, time_scale: float = 1.0, completed_at) -> Campaign:
    return Campaign(
        id=campaign_id, name=f"campaign-{campaign_id}", target_id="t-1",
        window_seconds=86400, time_scale=time_scale,
        plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
        status=CampaignStatus.COMPLETED, completed_at=completed_at,
    )


def _seed_config(session) -> None:
    from agenteval.storage.db import ModelConfigDB

    ModelConfigRepository(session).create(ModelConfigDB(
        id="mc-1", name="analysis-cfg", provider="openai_compatible", capability="chat",
        endpoint_url="https://models.example.com/v1/chat/completions", model_name="m",
        is_analysis_default=True,
    ))


def _seed_baseline(session, completed_at) -> None:
    CampaignRepository(session).create(_campaign("camp-base", completed_at=completed_at))
    row = CampaignAnalysisDB(campaign_id="camp-base", status="completed")
    row.set_result({"overall": "上期达标", "problems": [], "scenario_narratives": [], "suggestions": []})
    session.add(row)
    session.commit()


def _seed_current(session, *, time_scale: float = 1.0, completed_at) -> None:
    CampaignRepository(session).create(
        _campaign("camp-cur", time_scale=time_scale, completed_at=completed_at)
    )
    RunRepository(session).create(EvalRun(
        id="run-cur", target_id="t-1", scenario_id="s-1", campaign_id="camp-cur",
        status=RunStatus.COMPLETED, started_at=utc_now(),
        summary=RunSummary(total_cases=2, pass_rate=1.0, avg_latency_ms=300),
    ))


@pytest.fixture()
def comparison_spy(monkeypatch):
    calls: list[tuple[str, str]] = []

    def _spy(campaign_id, *, triggered_by, baseline_campaign_id=None):
        calls.append((campaign_id, triggered_by))

    monkeypatch.setattr(comparison_module, "start_campaign_comparison", _spy)
    return calls


async def test_production_line_with_baseline_enqueues_comparison(db_session, monkeypatch, comparison_spy):
    monkeypatch.setattr(analysis_module, "get_session", lambda: db_session)
    _seed_config(db_session)
    _seed_baseline(db_session, T0)
    _seed_current(db_session, completed_at=T0 + timedelta(hours=2))

    await execute_campaign_analysis(
        "camp-cur", triggered_by="auto", chat_client=FakeChatClient(STAGE1, STAGE2),
    )

    analysis_row = db_session.exec(
        select(CampaignAnalysisDB).where(CampaignAnalysisDB.campaign_id == "camp-cur")
    ).one()
    assert analysis_row.status == "completed"
    assert comparison_spy == [("camp-cur", "auto")]


async def test_accelerated_line_does_not_enqueue(db_session, monkeypatch, comparison_spy):
    monkeypatch.setattr(analysis_module, "get_session", lambda: db_session)
    _seed_config(db_session)
    _seed_baseline(db_session, T0)
    _seed_current(db_session, time_scale=24.0, completed_at=T0 + timedelta(hours=2))

    await execute_campaign_analysis(
        "camp-cur", triggered_by="auto", chat_client=FakeChatClient(STAGE1, STAGE2),
    )

    assert comparison_spy == []


async def test_no_auto_baseline_does_not_enqueue(db_session, monkeypatch, comparison_spy):
    monkeypatch.setattr(analysis_module, "get_session", lambda: db_session)
    _seed_config(db_session)
    _seed_current(db_session, completed_at=T0 + timedelta(hours=2))  # 无历史活动

    await execute_campaign_analysis(
        "camp-cur", triggered_by="auto", chat_client=FakeChatClient(STAGE1, STAGE2),
    )

    assert comparison_spy == []


async def test_missing_model_skips_comparison(db_session, monkeypatch, comparison_spy):
    monkeypatch.setattr(analysis_module, "get_session", lambda: db_session)
    _seed_config(db_session)
    _seed_baseline(db_session, T0)
    _seed_current(db_session, completed_at=T0 + timedelta(hours=2))

    # 分析本体解析得到模型（成功），自动链再次解析时模型已不可用 → 静默跳过
    calls = {"n": 0}

    def _resolve(campaign, session):
        calls["n"] += 1
        return SimpleNamespace(id="mc-1") if calls["n"] == 1 else None

    monkeypatch.setattr(analysis_module, "resolve_analysis_model", _resolve)

    await execute_campaign_analysis(
        "camp-cur", triggered_by="auto", chat_client=FakeChatClient(STAGE1, STAGE2),
    )

    analysis_row = db_session.exec(
        select(CampaignAnalysisDB).where(CampaignAnalysisDB.campaign_id == "camp-cur")
    ).one()
    assert analysis_row.status == "completed"
    assert comparison_spy == []


async def test_comparison_enqueue_error_does_not_affect_analysis(db_session, monkeypatch):
    monkeypatch.setattr(analysis_module, "get_session", lambda: db_session)
    _seed_config(db_session)
    _seed_baseline(db_session, T0)
    _seed_current(db_session, completed_at=T0 + timedelta(hours=2))

    def _boom(campaign_id, *, triggered_by, baseline_campaign_id=None):
        raise RuntimeError("enqueue failed")

    monkeypatch.setattr(comparison_module, "start_campaign_comparison", _boom)

    await execute_campaign_analysis(
        "camp-cur", triggered_by="auto", chat_client=FakeChatClient(STAGE1, STAGE2),
    )

    analysis_row = db_session.exec(
        select(CampaignAnalysisDB).where(CampaignAnalysisDB.campaign_id == "camp-cur")
    ).one()
    assert analysis_row.status == "completed"  # 异常不冒泡，分析结果不受影响