"""Auto-chain tests: analysis completed → period comparison enqueued (v0.8 ticket 04). Only 正式线 campaigns with a resolvable model and an auto baseline enqueue the comparison task; all other branches skip silently without touching the analysis row. The comparison entry point is spied, not executed. """ import json from datetime import timedelta from types import SimpleNamespace import pytest from agenteval.evaluation import analysis as analysis_module from agenteval.evaluation import comparison as comparison_module from agenteval.evaluation.analysis import execute_campaign_analysis from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus, RunSummary from agenteval.storage.db import CampaignAnalysisDB, utc_now from agenteval.storage.model_config_repository import ModelConfigRepository from agenteval.storage.repository import CampaignRepository, RunRepository from sqlmodel import select T0 = utc_now().replace(tzinfo=None) - timedelta(hours=3) STAGE1 = json.dumps({"narrative": "售前场景表现稳定", "problems": []}) STAGE2 = json.dumps({"overall": "整体达标", "problems": [], "suggestions": []}) class FakeChatClient: def __init__(self, *responses): self._responses = list(responses) async def __call__(self, messages: list[dict]) -> str: return self._responses.pop(0) def _campaign(campaign_id: str, *, time_scale: float = 1.0, completed_at) -> Campaign: return Campaign( id=campaign_id, name=f"campaign-{campaign_id}", target_id="t-1", window_seconds=86400, time_scale=time_scale, plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)], status=CampaignStatus.COMPLETED, completed_at=completed_at, ) def _seed_config(session) -> None: from agenteval.storage.db import ModelConfigDB ModelConfigRepository(session).create(ModelConfigDB( id="mc-1", name="analysis-cfg", provider="openai_compatible", capability="chat", endpoint_url="https://models.example.com/v1/chat/completions", model_name="m", is_analysis_default=True, )) def _seed_baseline(session, completed_at) -> None: CampaignRepository(session).create(_campaign("camp-base", completed_at=completed_at)) row = CampaignAnalysisDB(campaign_id="camp-base", status="completed") row.set_result({"overall": "上期达标", "problems": [], "scenario_narratives": [], "suggestions": []}) session.add(row) session.commit() def _seed_current(session, *, time_scale: float = 1.0, completed_at) -> None: CampaignRepository(session).create( _campaign("camp-cur", time_scale=time_scale, completed_at=completed_at) ) RunRepository(session).create(EvalRun( id="run-cur", target_id="t-1", scenario_id="s-1", campaign_id="camp-cur", status=RunStatus.COMPLETED, started_at=utc_now(), summary=RunSummary(total_cases=2, pass_rate=1.0, avg_latency_ms=300), )) @pytest.fixture() def comparison_spy(monkeypatch): calls: list[tuple[str, str]] = [] def _spy(campaign_id, *, triggered_by, baseline_campaign_id=None): calls.append((campaign_id, triggered_by)) monkeypatch.setattr(comparison_module, "start_campaign_comparison", _spy) return calls async def test_production_line_with_baseline_enqueues_comparison(db_session, monkeypatch, comparison_spy): monkeypatch.setattr(analysis_module, "get_session", lambda: db_session) _seed_config(db_session) _seed_baseline(db_session, T0) _seed_current(db_session, completed_at=T0 + timedelta(hours=2)) await execute_campaign_analysis( "camp-cur", triggered_by="auto", chat_client=FakeChatClient(STAGE1, STAGE2), ) analysis_row = db_session.exec( select(CampaignAnalysisDB).where(CampaignAnalysisDB.campaign_id == "camp-cur") ).one() assert analysis_row.status == "completed" assert comparison_spy == [("camp-cur", "auto")] async def test_accelerated_line_does_not_enqueue(db_session, monkeypatch, comparison_spy): monkeypatch.setattr(analysis_module, "get_session", lambda: db_session) _seed_config(db_session) _seed_baseline(db_session, T0) _seed_current(db_session, time_scale=24.0, completed_at=T0 + timedelta(hours=2)) await execute_campaign_analysis( "camp-cur", triggered_by="auto", chat_client=FakeChatClient(STAGE1, STAGE2), ) assert comparison_spy == [] async def test_no_auto_baseline_does_not_enqueue(db_session, monkeypatch, comparison_spy): monkeypatch.setattr(analysis_module, "get_session", lambda: db_session) _seed_config(db_session) _seed_current(db_session, completed_at=T0 + timedelta(hours=2)) # 无历史活动 await execute_campaign_analysis( "camp-cur", triggered_by="auto", chat_client=FakeChatClient(STAGE1, STAGE2), ) assert comparison_spy == [] async def test_missing_model_skips_comparison(db_session, monkeypatch, comparison_spy): monkeypatch.setattr(analysis_module, "get_session", lambda: db_session) _seed_config(db_session) _seed_baseline(db_session, T0) _seed_current(db_session, completed_at=T0 + timedelta(hours=2)) # 分析本体解析得到模型(成功),自动链再次解析时模型已不可用 → 静默跳过 calls = {"n": 0} def _resolve(campaign, session): calls["n"] += 1 return SimpleNamespace(id="mc-1") if calls["n"] == 1 else None monkeypatch.setattr(analysis_module, "resolve_analysis_model", _resolve) await execute_campaign_analysis( "camp-cur", triggered_by="auto", chat_client=FakeChatClient(STAGE1, STAGE2), ) analysis_row = db_session.exec( select(CampaignAnalysisDB).where(CampaignAnalysisDB.campaign_id == "camp-cur") ).one() assert analysis_row.status == "completed" assert comparison_spy == [] async def test_comparison_enqueue_error_does_not_affect_analysis(db_session, monkeypatch): monkeypatch.setattr(analysis_module, "get_session", lambda: db_session) _seed_config(db_session) _seed_baseline(db_session, T0) _seed_current(db_session, completed_at=T0 + timedelta(hours=2)) def _boom(campaign_id, *, triggered_by, baseline_campaign_id=None): raise RuntimeError("enqueue failed") monkeypatch.setattr(comparison_module, "start_campaign_comparison", _boom) await execute_campaign_analysis( "camp-cur", triggered_by="auto", chat_client=FakeChatClient(STAGE1, STAGE2), ) analysis_row = db_session.exec( select(CampaignAnalysisDB).where(CampaignAnalysisDB.campaign_id == "camp-cur") ).one() assert analysis_row.status == "completed" # 异常不冒泡,分析结果不受影响