"""Characterization tests: intelligence-job settlement contract (Phase 2.8). 锁定两个 executor(智能分析 / 周期对比)的结算契约边界,供 ``execute(job_kind, work_fn)`` 接缝收敛(Phase 2.9–2.11)对照: 1. 认领竞争:两个执行流观察同一 queued 行,只有一个真正跑领域工作。 2. 重复触发:completed 行被再次触发时重置回 queued 并重跑;generating 中的行不被回退(后者已有 test_reenqueue_does_not_move_generating_*)。 3. 失败截断:work 异常落账时 error 截断到 500 字符(一处语义)。 4. 恢复上限:``recover_campaign_intelligence_jobs`` 把超过恢复次数上限 的 queued 行落 failed,不再重启。 不改动任何生产代码。 """ import asyncio import json from datetime import timedelta import pytest from agenteval.evaluation import intelligence_jobs from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus, RunSummary from agenteval.storage.db import ( CampaignAnalysisDB, ModelConfigDB, utc_now, ) from agenteval.storage.model_config_repository import ModelConfigRepository from agenteval.storage.async_job_repository import CampaignAnalysisRepository, CampaignPeriodComparisonRepository from agenteval.storage.repository import ( CampaignRepository, RunRepository, ) from sqlmodel import Session, SQLModel, create_engine T0 = utc_now().replace(tzinfo=None) - timedelta(hours=1) STAGE1 = json.dumps({"narrative": "n", "problems": []}) STAGE2 = json.dumps({"overall": "o", "problems": [], "suggestions": []}) NARRATION = json.dumps({"trend": "stable", "summary": "s"}) @pytest.fixture() def db_session(tmp_path): from agenteval.storage.db import ( # noqa: F401 CampaignDB, ) engine = create_engine( f"sqlite:///{tmp_path / 'settlement.db'}", connect_args={"check_same_thread": False}, ) SQLModel.metadata.create_all(engine) session = Session(engine) try: yield session finally: session.close() engine.dispose() class FakeChatClient: def __init__(self, *responses): self._responses = list(responses) self.calls = 0 async def __call__(self, messages): self.calls += 1 if not self._responses: raise AssertionError("unexpected extra LLM call") item = self._responses.pop(0) if isinstance(item, Exception): raise item return item def _campaign( campaign_id: str = "camp-1", *, time_scale: float = 1.0, status: CampaignStatus = CampaignStatus.COMPLETED, completed_at=T0 + timedelta(hours=1), ) -> Campaign: return Campaign( id=campaign_id, name=f"campaign-{campaign_id}", target_id="t-1", window_seconds=86400, time_scale=time_scale, plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)], status=status, completed_at=completed_at, ) def _seed_config(session) -> None: ModelConfigRepository(session).create( ModelConfigDB( id="mc-default", name="cfg", provider="openai_compatible", capability="chat", endpoint_url="https://models.example.com/v1/chat/completions", model_name="m", is_analysis_default=True, ) ) def _seed_failed_run(session, campaign_id: str = "camp-1") -> None: from agenteval.storage.db import EvalResultDB, TurnDB RunRepository(session).create( EvalRun( id=f"run-{campaign_id}", target_id="t-1", scenario_id="s-1", campaign_id=campaign_id, status=RunStatus.COMPLETED, ) ) turn = TurnDB(id=f"run-{campaign_id}-turn-0", run_id=f"run-{campaign_id}", case_id="c0", round_index=0) turn.set_sent_message({"msgBody": {"content": "用户消息"}}) turn.set_reply({"msgBody": {"content": "答"}}) session.add(turn) session.add( EvalResultDB( run_id=f"run-{campaign_id}", case_id="c0", turn_id=turn.id, rule_type="llm_score", passed=False, reason="不合格", ) ) session.commit() class TestClaimRace: """两个执行流竞争同一 queued 行:只有一个跑领域工作。""" async def test_concurrent_analysis_jobs_run_work_once(self, db_session): _seed_config(db_session) CampaignRepository(db_session).create(_campaign()) _seed_failed_run(db_session) CampaignAnalysisRepository(db_session).enqueue("camp-1", triggered_by="auto") client = FakeChatClient(STAGE1, STAGE2) await asyncio.gather( intelligence_jobs.execute_campaign_analysis_job( "camp-1", triggered_by="auto", chat_client=client, session_factory=lambda: db_session, ), intelligence_jobs.execute_campaign_analysis_job( "camp-1", triggered_by="auto", chat_client=client, session_factory=lambda: db_session, ), ) assert client.calls == 2 # 阶段一 1 场景 + 阶段二 1 次 = 恰好一次完整工作 row = CampaignAnalysisRepository(db_session).get_by_campaign("camp-1") assert row.status == "completed" async def test_generating_row_blocks_second_executor(self, db_session): """已在 generating 的行再次进入 executor:不认领、不跑工作。""" _seed_config(db_session) CampaignRepository(db_session).create(_campaign()) CampaignAnalysisRepository(db_session).upsert("camp-1", status="generating", triggered_by="auto") client = FakeChatClient(STAGE1, STAGE2) await intelligence_jobs.execute_campaign_analysis_job( "camp-1", triggered_by="auto", chat_client=client, session_factory=lambda: db_session, ) assert client.calls == 0 assert CampaignAnalysisRepository(db_session).get_by_campaign("camp-1").status == "generating" class TestRetriggerSemantics: """completed 行再触发 → 重置 queued 并重跑;queued 重复入队保持原样。""" def test_retrigger_of_completed_campaign_resets_and_relaunches(self, db_session, monkeypatch): """重复触发幂等观察:completed 行经 enqueue 重置回 queued 并重启 worker。""" repo = CampaignAnalysisRepository(db_session) repo.upsert("camp-1", status="completed", result={"overall": "旧结论"}, triggered_by="auto") launched = [] monkeypatch.setattr( intelligence_jobs, "_launch_analysis", lambda cid, *, triggered_by: launched.append((cid, triggered_by)), ) intelligence_jobs.enqueue_campaign_analysis("camp-1", triggered_by="manual", session=db_session) row = repo.get_by_campaign("camp-1") assert row.status == "queued" assert row.result is None assert launched == [("camp-1", "manual")] def test_reenqueue_completed_row_resets_to_queued(self, db_session): repo = CampaignAnalysisRepository(db_session) repo.upsert("camp-1", status="completed", result={"overall": "旧"}, triggered_by="auto") row = repo.enqueue("camp-1", triggered_by="manual") assert row.status == "queued" assert row.result is None assert row.triggered_by == "manual" def test_reenqueue_completed_comparison_resets_and_replaces_baseline(self, db_session): repo = CampaignPeriodComparisonRepository(db_session) repo.upsert( "camp-cur", status="completed", baseline_campaign_id="camp-old", result={"trend": "stable"}, triggered_by="auto", ) row = repo.enqueue("camp-cur", baseline_campaign_id="camp-new", triggered_by="manual") assert row.status == "queued" assert row.baseline_campaign_id == "camp-new" assert row.result is None class TestFailureTruncation: """work 异常落账:error 截断到 500 字符。""" async def test_analysis_error_truncated_to_500(self, db_session): _seed_config(db_session) CampaignRepository(db_session).create(_campaign()) _seed_failed_run(db_session) client = FakeChatClient(RuntimeError("长" * 800)) await intelligence_jobs.execute_campaign_analysis_job( "camp-1", triggered_by="auto", chat_client=client, session_factory=lambda: db_session, ) row = CampaignAnalysisRepository(db_session).get_by_campaign("camp-1") assert row.status == "failed" assert len(row.error) == 500 async def test_comparison_error_truncated_to_500(self, db_session): _seed_config(db_session) baseline, current = _campaign("camp-base"), _campaign("camp-cur") CampaignRepository(db_session).create(baseline) CampaignRepository(db_session).create(current) for cid in ("camp-base", "camp-cur"): analysis = CampaignAnalysisDB(campaign_id=cid, status="completed") analysis.set_result({"overall": cid}) db_session.add(analysis) RunRepository(db_session).create( EvalRun( id=f"run-{cid}", target_id="t-1", scenario_id="s-1", campaign_id=cid, status=RunStatus.COMPLETED, started_at=utc_now(), summary=RunSummary(total_cases=2, pass_rate=0.5, avg_latency_ms=700), ) ) db_session.commit() client = FakeChatClient(RuntimeError("长" * 800)) await intelligence_jobs.execute_campaign_comparison_job( "camp-cur", triggered_by="manual", baseline_campaign_id="camp-base", chat_client=client, session_factory=lambda: db_session, ) row = CampaignPeriodComparisonRepository(db_session).get_by_campaign("camp-cur") assert row.status == "failed" assert len(row.error) == 500 class TestRecoveryCap: """recover_campaign_intelligence_jobs:超限落 failed,中断行计入返回。""" def test_exhausted_queued_job_fails_instead_of_relaunch(self, db_session, monkeypatch): repo = CampaignAnalysisRepository(db_session) repo.enqueue("camp-1", triggered_by="auto") launched = [] monkeypatch.setattr( intelligence_jobs, "_launch_analysis", lambda cid, *, triggered_by: launched.append(cid), ) for _ in range(intelligence_jobs.MAX_QUEUED_RECOVERY_ATTEMPTS): intelligence_jobs.recover_campaign_intelligence_jobs(db_session) assert launched == ["camp-1"] * intelligence_jobs.MAX_QUEUED_RECOVERY_ATTEMPTS interrupted, relaunched = intelligence_jobs.recover_campaign_intelligence_jobs(db_session) assert relaunched == 0 assert launched == ["camp-1"] * intelligence_jobs.MAX_QUEUED_RECOVERY_ATTEMPTS row = repo.get_by_campaign("camp-1") assert row.status == "failed" assert "上限" in row.error def test_recovery_reports_interrupted_generating_rows(self, db_session, monkeypatch): CampaignAnalysisRepository(db_session).upsert("camp-a", status="generating", triggered_by="auto") CampaignPeriodComparisonRepository(db_session).upsert( "camp-c", status="generating", baseline_campaign_id="camp-base", triggered_by="auto", ) monkeypatch.setattr(intelligence_jobs, "_launch_analysis", lambda *a, **kw: None) monkeypatch.setattr(intelligence_jobs, "_launch_comparison", lambda *a, **kw: None) interrupted, relaunched = intelligence_jobs.recover_campaign_intelligence_jobs(db_session) assert interrupted == 2 assert relaunched == 0 assert CampaignAnalysisRepository(db_session).get_by_campaign("camp-a").status == "failed" assert CampaignPeriodComparisonRepository(db_session).get_by_campaign("camp-c").status == "failed"