AgentEvalTool/tests/unit/test_intelligence_jobs_settlement_characterization.py

"""Characterization tests: intelligence-job settlement contract (Phase 2.8).

锁定两个 executor（智能分析 / 周期对比）的结算契约边界，供
``execute(job_kind, work_fn)`` 接缝收敛（Phase 2.9–2.11）对照：

1. 认领竞争：两个执行流观察同一 queued 行，只有一个真正跑领域工作。
2. 重复触发：completed 行被再次触发时重置回 queued 并重跑；generating
   中的行不被回退（后者已有 test_reenqueue_does_not_move_generating_*）。
3. 失败截断：work 异常落账时 error 截断到 500 字符（一处语义）。
4. 恢复上限：``recover_campaign_intelligence_jobs`` 把超过恢复次数上限
   的 queued 行落 failed，不再重启。

不改动任何生产代码。
"""

import asyncio
import json
from datetime import timedelta

import pytest
from agenteval.evaluation import intelligence_jobs
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus, RunSummary
from agenteval.storage.db import (
    CampaignAnalysisDB,
    ModelConfigDB,
    utc_now,
)
from agenteval.storage.model_config_repository import ModelConfigRepository
from agenteval.storage.async_job_repository import CampaignAnalysisRepository, CampaignPeriodComparisonRepository
from agenteval.storage.repository import (
    CampaignRepository,
    RunRepository,
)
from sqlmodel import Session, SQLModel, create_engine

T0 = utc_now().replace(tzinfo=None) - timedelta(hours=1)

STAGE1 = json.dumps({"narrative": "n", "problems": []})
STAGE2 = json.dumps({"overall": "o", "problems": [], "suggestions": []})
NARRATION = json.dumps({"trend": "stable", "summary": "s"})


@pytest.fixture()
def db_session(tmp_path):
    from agenteval.storage.db import (  # noqa: F401
        CampaignDB,
    )

    engine = create_engine(
        f"sqlite:///{tmp_path / 'settlement.db'}",
        connect_args={"check_same_thread": False},
    )
    SQLModel.metadata.create_all(engine)
    session = Session(engine)
    try:
        yield session
    finally:
        session.close()
        engine.dispose()


class FakeChatClient:
    def __init__(self, *responses):
        self._responses = list(responses)
        self.calls = 0

    async def __call__(self, messages):
        self.calls += 1
        if not self._responses:
            raise AssertionError("unexpected extra LLM call")
        item = self._responses.pop(0)
        if isinstance(item, Exception):
            raise item
        return item


def _campaign(
    campaign_id: str = "camp-1",
    *,
    time_scale: float = 1.0,
    status: CampaignStatus = CampaignStatus.COMPLETED,
    completed_at=T0 + timedelta(hours=1),
) -> Campaign:
    return Campaign(
        id=campaign_id,
        name=f"campaign-{campaign_id}",
        target_id="t-1",
        window_seconds=86400,
        time_scale=time_scale,
        plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)],
        status=status,
        completed_at=completed_at,
    )


def _seed_config(session) -> None:
    ModelConfigRepository(session).create(
        ModelConfigDB(
            id="mc-default",
            name="cfg",
            provider="openai_compatible",
            capability="chat",
            endpoint_url="https://models.example.com/v1/chat/completions",
            model_name="m",
            is_analysis_default=True,
        )
    )


def _seed_failed_run(session, campaign_id: str = "camp-1") -> None:
    from agenteval.storage.db import EvalResultDB, TurnDB

    RunRepository(session).create(
        EvalRun(
            id=f"run-{campaign_id}",
            target_id="t-1",
            scenario_id="s-1",
            campaign_id=campaign_id,
            status=RunStatus.COMPLETED,
        )
    )
    turn = TurnDB(id=f"run-{campaign_id}-turn-0", run_id=f"run-{campaign_id}", case_id="c0", round_index=0)
    turn.set_sent_message({"msgBody": {"content": "用户消息"}})
    turn.set_reply({"msgBody": {"content": "答"}})
    session.add(turn)
    session.add(
        EvalResultDB(
            run_id=f"run-{campaign_id}",
            case_id="c0",
            turn_id=turn.id,
            rule_type="llm_score",
            passed=False,
            reason="不合格",
        )
    )
    session.commit()


class TestClaimRace:
    """两个执行流竞争同一 queued 行：只有一个跑领域工作。"""

    async def test_concurrent_analysis_jobs_run_work_once(self, db_session):
        _seed_config(db_session)
        CampaignRepository(db_session).create(_campaign())
        _seed_failed_run(db_session)
        CampaignAnalysisRepository(db_session).enqueue("camp-1", triggered_by="auto")

        client = FakeChatClient(STAGE1, STAGE2)
        await asyncio.gather(
            intelligence_jobs.execute_campaign_analysis_job(
                "camp-1",
                triggered_by="auto",
                chat_client=client,
                session_factory=lambda: db_session,
            ),
            intelligence_jobs.execute_campaign_analysis_job(
                "camp-1",
                triggered_by="auto",
                chat_client=client,
                session_factory=lambda: db_session,
            ),
        )

        assert client.calls == 2  # 阶段一 1 场景 + 阶段二 1 次 = 恰好一次完整工作
        row = CampaignAnalysisRepository(db_session).get_by_campaign("camp-1")
        assert row.status == "completed"

    async def test_generating_row_blocks_second_executor(self, db_session):
        """已在 generating 的行再次进入 executor：不认领、不跑工作。"""
        _seed_config(db_session)
        CampaignRepository(db_session).create(_campaign())
        CampaignAnalysisRepository(db_session).upsert("camp-1", status="generating", triggered_by="auto")

        client = FakeChatClient(STAGE1, STAGE2)
        await intelligence_jobs.execute_campaign_analysis_job(
            "camp-1",
            triggered_by="auto",
            chat_client=client,
            session_factory=lambda: db_session,
        )

        assert client.calls == 0
        assert CampaignAnalysisRepository(db_session).get_by_campaign("camp-1").status == "generating"


class TestRetriggerSemantics:
    """completed 行再触发 → 重置 queued 并重跑；queued 重复入队保持原样。"""

    def test_retrigger_of_completed_campaign_resets_and_relaunches(self, db_session, monkeypatch):
        """重复触发幂等观察：completed 行经 enqueue 重置回 queued 并重启 worker。"""
        repo = CampaignAnalysisRepository(db_session)
        repo.upsert("camp-1", status="completed", result={"overall": "旧结论"}, triggered_by="auto")
        launched = []
        monkeypatch.setattr(
            intelligence_jobs,
            "_launch_analysis",
            lambda cid, *, triggered_by: launched.append((cid, triggered_by)),
        )

        intelligence_jobs.enqueue_campaign_analysis("camp-1", triggered_by="manual", session=db_session)

        row = repo.get_by_campaign("camp-1")
        assert row.status == "queued"
        assert row.result is None
        assert launched == [("camp-1", "manual")]

    def test_reenqueue_completed_row_resets_to_queued(self, db_session):
        repo = CampaignAnalysisRepository(db_session)
        repo.upsert("camp-1", status="completed", result={"overall": "旧"}, triggered_by="auto")

        row = repo.enqueue("camp-1", triggered_by="manual")

        assert row.status == "queued"
        assert row.result is None
        assert row.triggered_by == "manual"

    def test_reenqueue_completed_comparison_resets_and_replaces_baseline(self, db_session):
        repo = CampaignPeriodComparisonRepository(db_session)
        repo.upsert(
            "camp-cur",
            status="completed",
            baseline_campaign_id="camp-old",
            result={"trend": "stable"},
            triggered_by="auto",
        )

        row = repo.enqueue("camp-cur", baseline_campaign_id="camp-new", triggered_by="manual")

        assert row.status == "queued"
        assert row.baseline_campaign_id == "camp-new"
        assert row.result is None


class TestFailureTruncation:
    """work 异常落账：error 截断到 500 字符。"""

    async def test_analysis_error_truncated_to_500(self, db_session):
        _seed_config(db_session)
        CampaignRepository(db_session).create(_campaign())
        _seed_failed_run(db_session)

        client = FakeChatClient(RuntimeError("长" * 800))
        await intelligence_jobs.execute_campaign_analysis_job(
            "camp-1",
            triggered_by="auto",
            chat_client=client,
            session_factory=lambda: db_session,
        )

        row = CampaignAnalysisRepository(db_session).get_by_campaign("camp-1")
        assert row.status == "failed"
        assert len(row.error) == 500

    async def test_comparison_error_truncated_to_500(self, db_session):
        _seed_config(db_session)
        baseline, current = _campaign("camp-base"), _campaign("camp-cur")
        CampaignRepository(db_session).create(baseline)
        CampaignRepository(db_session).create(current)
        for cid in ("camp-base", "camp-cur"):
            analysis = CampaignAnalysisDB(campaign_id=cid, status="completed")
            analysis.set_result({"overall": cid})
            db_session.add(analysis)
            RunRepository(db_session).create(
                EvalRun(
                    id=f"run-{cid}",
                    target_id="t-1",
                    scenario_id="s-1",
                    campaign_id=cid,
                    status=RunStatus.COMPLETED,
                    started_at=utc_now(),
                    summary=RunSummary(total_cases=2, pass_rate=0.5, avg_latency_ms=700),
                )
            )
        db_session.commit()

        client = FakeChatClient(RuntimeError("长" * 800))
        await intelligence_jobs.execute_campaign_comparison_job(
            "camp-cur",
            triggered_by="manual",
            baseline_campaign_id="camp-base",
            chat_client=client,
            session_factory=lambda: db_session,
        )

        row = CampaignPeriodComparisonRepository(db_session).get_by_campaign("camp-cur")
        assert row.status == "failed"
        assert len(row.error) == 500


class TestRecoveryCap:
    """recover_campaign_intelligence_jobs：超限落 failed，中断行计入返回。"""

    def test_exhausted_queued_job_fails_instead_of_relaunch(self, db_session, monkeypatch):
        repo = CampaignAnalysisRepository(db_session)
        repo.enqueue("camp-1", triggered_by="auto")
        launched = []
        monkeypatch.setattr(
            intelligence_jobs,
            "_launch_analysis",
            lambda cid, *, triggered_by: launched.append(cid),
        )

        for _ in range(intelligence_jobs.MAX_QUEUED_RECOVERY_ATTEMPTS):
            intelligence_jobs.recover_campaign_intelligence_jobs(db_session)
        assert launched == ["camp-1"] * intelligence_jobs.MAX_QUEUED_RECOVERY_ATTEMPTS

        interrupted, relaunched = intelligence_jobs.recover_campaign_intelligence_jobs(db_session)
        assert relaunched == 0
        assert launched == ["camp-1"] * intelligence_jobs.MAX_QUEUED_RECOVERY_ATTEMPTS
        row = repo.get_by_campaign("camp-1")
        assert row.status == "failed"
        assert "上限" in row.error

    def test_recovery_reports_interrupted_generating_rows(self, db_session, monkeypatch):
        CampaignAnalysisRepository(db_session).upsert("camp-a", status="generating", triggered_by="auto")
        CampaignPeriodComparisonRepository(db_session).upsert(
            "camp-c",
            status="generating",
            baseline_campaign_id="camp-base",
            triggered_by="auto",
        )
        monkeypatch.setattr(intelligence_jobs, "_launch_analysis", lambda *a, **kw: None)
        monkeypatch.setattr(intelligence_jobs, "_launch_comparison", lambda *a, **kw: None)

        interrupted, relaunched = intelligence_jobs.recover_campaign_intelligence_jobs(db_session)

        assert interrupted == 2
        assert relaunched == 0
        assert CampaignAnalysisRepository(db_session).get_by_campaign("camp-a").status == "failed"
        assert CampaignPeriodComparisonRepository(db_session).get_by_campaign("camp-c").status == "failed"