"""Unit tests for the RunSummary value type and the single cross-run
aggregation seam (ADR-0004: faults count 0.0, user-cancelled excluded)."""

from agenteval.evaluation.metrics import aggregate_runs
from agenteval.models import EvalRun, RunStatus, RunSummary

# ── RunSummary parsing (legacy dict rows must keep parsing) ────────────────


def test_parses_full_engine_summary():
    s = RunSummary.model_validate({
        "total_cases": 3,
        "passed_cases": 2,
        "failed_cases": 1,
        "total_rules": 5,
        "passed_rules": 4,
        "pass_rate": 0.6667,
        "avg_latency_ms": 123.4,
        "case_outcomes": {"c1": {"passed": True, "connectivity": False}},
        "case_errors": [{"case_id": "c2", "stage": "generate_messages", "error": "boom"}],
        "model_configs": {"judge": {"model": "m"}},
    })
    assert s.pass_rate == 0.6667
    assert s.avg_latency_ms == 123.4
    assert s.case_outcomes["c1"].passed is True
    assert s.case_outcomes["c1"].connectivity is False
    assert s.error is None


def test_parses_legacy_string_error():
    s = RunSummary.model_validate({"error": "channel exploded"})
    assert s.error is not None
    assert s.error.code == "error"
    assert s.error.message == "channel exploded"
    assert s.is_cancelled is False


def test_parses_object_error_and_detects_cancel():
    s = RunSummary.model_validate({"error": {"code": "cancelled_by_user", "message": "评测已手动停止"}})
    assert s.error is not None
    assert s.error.code == "cancelled_by_user"
    assert s.is_cancelled is True


def test_missing_keys_default_and_unknown_keys_tolerated():
    s = RunSummary.model_validate({"pass_rate": 0.5, "some_future_key": 1})
    assert s.pass_rate == 0.5
    assert s.total_cases == 0
    assert s.case_outcomes == {}
    assert s.error is None


def test_eval_run_summary_field_is_typed():
    run = EvalRun(target_id="t", scenario_id="s", summary={"pass_rate": 1.0})
    assert isinstance(run.summary, RunSummary)
    assert run.summary.pass_rate == 1.0


# ── aggregate_runs (ADR-0004) ──────────────────────────────────────────────


def _run(status, *, pass_rate=None, latency=None, error=None) -> EvalRun:
    summary = None
    if pass_rate is not None or latency is not None or error is not None:
        summary = {}
        if pass_rate is not None:
            summary["pass_rate"] = pass_rate
        if latency is not None:
            summary["avg_latency_ms"] = latency
        if error is not None:
            summary["error"] = error
    return EvalRun(target_id="t", scenario_id="s", status=status, summary=summary)


def test_empty_runs_aggregate_to_none():
    agg = aggregate_runs([])
    assert agg == {"run_count": 0, "pass_rate": None, "availability": None, "avg_latency_ms": None}


def test_completed_runs_average_pass_rate_and_latency():
    agg = aggregate_runs([
        _run(RunStatus.COMPLETED, pass_rate=1.0, latency=100),
        _run(RunStatus.COMPLETED, pass_rate=0.5, latency=200),
    ])
    assert agg["run_count"] == 2
    assert agg["pass_rate"] == 0.75
    assert agg["availability"] == 1.0
    assert agg["avg_latency_ms"] == 150.0


def test_faulted_run_counts_zero_in_both_denominators():
    # ADR-0004: a genuine execution fault drags pass_rate AND availability down.
    agg = aggregate_runs([
        _run(RunStatus.COMPLETED, pass_rate=1.0),
        _run(RunStatus.FAILED, error="channel exploded"),
    ])
    assert agg["pass_rate"] == 0.5
    assert agg["availability"] == 0.5


def test_cancelled_run_excluded_from_denominators():
    # ADR-0004: user cancellation is neither a quality nor availability signal.
    agg = aggregate_runs([
        _run(RunStatus.COMPLETED, pass_rate=1.0),
        _run(RunStatus.FAILED, error={"code": "cancelled_by_user", "message": "stop"}),
    ])
    assert agg["run_count"] == 2          # what happened stays visible
    assert agg["pass_rate"] == 1.0        # cancelled run out of the denominator
    assert agg["availability"] == 1.0


def test_all_cancelled_aggregates_to_none():
    agg = aggregate_runs([
        _run(RunStatus.FAILED, error={"code": "cancelled_by_user", "message": "stop"}),
    ])
    assert agg["run_count"] == 1
    assert agg["pass_rate"] is None
    assert agg["availability"] is None
    assert agg["avg_latency_ms"] is None


def test_in_flight_runs_count_zero_not_cancelled():
    # pending/running (no summary) are not cancelled — they count 0.0 for now.
    agg = aggregate_runs([
        _run(RunStatus.RUNNING),
        _run(RunStatus.PENDING),
    ])
    assert agg["pass_rate"] == 0.0
    assert agg["availability"] == 0.0