AgentEvalTool/tests/unit/test_phase3_wiring.py

"""Phase 3 (v1.3.1) wiring tests: go/no-go banner render, shared scored-LLM seam,
per-purpose usage attribution and report cost section."""

import json
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from agenteval.evaluation.report_render import render_html, render_markdown
from agenteval.evaluation.rules.llm_score import LlmScoreRule
from agenteval.models import Case, CaseType, Turn

from tests.unit.test_phase2_wiring import report_seeded  # noqa: F401

# ── 3.1 go/no-go banner in rendered reports ──────────────────────────────


def _gng(decision: str) -> dict:
    return {
        "decision": decision,
        "summary": "判定型通过率未达标",
        "criteria_results": [{"passed": False, "detail": "判定型通过率 0% < 95%"}],
    }


def _report_dict(gng: dict | None) -> dict:
    return {
        "run_id": "r1",
        "target_name": "T",
        "target_id": "t",
        "scenario_name": "S",
        "scenario_id": "s",
        "started_at": "2026-08-25T00:00:00",
        "completed_at": "2026-08-25T00:10:00",
        "status": "completed",
        "summary": {
            "total_cases": 1,
            "passed_cases": 0,
            "failed_cases": 1,
            "total_rules": 1,
            "passed_rules": 0,
            "pass_rate": 0.0,
        },
        "go_no_go": gng,
        "cases": [],
    }


def test_html_banner_renders_no_go_decision():
    html = render_html(_report_dict(_gng("no_go")))
    assert 'class="verdict verdict-no_go"' in html
    assert "NO-GO — 不建议上线" in html
    assert "判定型通过率 0% < 95%" in html


def test_html_banner_renders_go_decision():
    html = render_html(_report_dict(_gng("go")))
    assert 'class="verdict verdict-go"' in html
    assert "GO — 建议上线" in html


def test_markdown_banner_renders_blockquote():
    md = render_markdown(_report_dict(_gng("conditional")))
    assert "> **上线评估：有条件通过 — 修复后复测**" in md
    assert "> ❌ 判定型通过率 0% < 95%" in md


def test_no_banner_when_go_no_go_absent():
    md = render_markdown(_report_dict(None))
    assert "上线评估" not in md
    html = render_html(_report_dict(None))
    assert 'class="verdict verdict-' not in html
    assert "上线评估" not in html


# ── 3.2 rule-level usage recording via chat_with_usage ──────────────────


@pytest.mark.asyncio
async def test_llm_score_rule_records_gateway_usage():
    rule = LlmScoreRule({"criteria": "礼貌", "min_score": 5})

    gateway = MagicMock()
    gateway.chat_with_usage = AsyncMock(
        return_value=(json.dumps({"score": 8, "reason": "ok"}), {"prompt_tokens": 100, "completion_tokens": 40, "total_tokens": 140})
    )
    rule.gateway = gateway
    rule.model_config = MagicMock()

    turn = Turn(
        id="t1", run_id="r1", case_id="c1", round_index=1,
        sent_message={"msgBody": {"content": "问题"}},
        reply={"msgBody": {"content": "回答"}},
        latency_ms=100,
    )
    result = await rule.evaluate(Case(id="c1", type=CaseType.SINGLE, messages=["x"]), [turn])
    assert result.passed is True
    assert rule.llm_usage == {"prompt_tokens": 100, "completion_tokens": 40, "total_tokens": 140}


@pytest.mark.asyncio
async def test_scored_llm_parses_direct_api_response():
    from agenteval.evaluation.rules.scored_llm import call_scored_llm

    mock_resp = MagicMock()
    mock_resp.raise_for_status = MagicMock()
    mock_resp.json = MagicMock(return_value={
        "choices": [{"message": {"content": json.dumps({"score": 9, "reason": "好"})}}]
    })
    with patch("agenteval.evaluation.rules.scored_llm.httpx.AsyncClient") as MockClient:
        instance = MockClient.return_value.__aenter__.return_value
        instance.post = AsyncMock(return_value=mock_resp)
        score, reason = await call_scored_llm("http://mock", None, "m", "sys", "user")
    assert score == 9
    assert reason == "好"


# ── 3.3 report cost section from recorded per-purpose usage ─────────────


@pytest.mark.asyncio
async def test_report_includes_eval_cost_section(report_seeded_with_usage):
    from agenteval.evaluation.report import generate_report

    session, run_id, _ = report_seeded_with_usage
    report = generate_report(run_id, session)
    cost = report["summary"]["eval_cost"]
    assert cost is not None
    judge = next(i for i in cost["by_purpose"] if i["purpose"] == "judge")
    assert judge["model_name"] == "gpt-4o-mini"
    assert judge["total_tokens"] == 1500
    assert cost["total_cost_usd"] is not None
    assert cost["total_tokens"] == 1500

    # Markdown 导出渲染成本表
    md = render_markdown(report)
    assert "## 评测成本" in md
    assert "gpt-4o-mini" in md


def test_report_cost_absent_without_usage(report_seeded):
    from agenteval.evaluation.report import generate_report

    session, run_id, _ = report_seeded
    report = generate_report(run_id, session)
    assert report["summary"]["eval_cost"] is None
    assert "评测成本" not in render_markdown(report)


@pytest.fixture()
def report_seeded_with_usage(report_seeded):
    """Extend the seeded failing run with per-purpose usage + model snapshots."""
    from agenteval.storage.repository import RunRepository

    session, run_id, scenario_id = report_seeded
    run = RunRepository(session).get(run_id)
    summary = dict(run.summary.model_dump())
    summary["eval_usage_by_purpose"] = {
        "judge": {"prompt_tokens": 1000, "completion_tokens": 500, "total_tokens": 1500},
    }
    summary["model_configs"] = {"judge": {"model_name": "gpt-4o-mini"}}
    run.summary = summary
    RunRepository(session).update(run)
    return session, run_id, scenario_id