AgentEvalTool/tests/unit/test_scenarios.py

"""Unit tests for scenarios loader and templates."""

import json
import tempfile
from pathlib import Path

import pytest
import yaml
from agenteval.scenarios.loader import ScenarioValidationError, load_scenario_file, validate_scenario
from agenteval.scenarios.templates import get_template, list_templates

# ── templates ─────────────────────────────────────────────────────────────

def test_list_templates_returns_list():
    tpls = list_templates()
    assert isinstance(tpls, list)
    assert len(tpls) >= 6


def test_template_has_required_fields():
    for tpl in list_templates():
        assert "id" in tpl
        assert "name" in tpl
        assert "description" in tpl
        assert "cases" in tpl
        assert isinstance(tpl["cases"], list)
        assert len(tpl["cases"]) > 0


def test_get_template_by_id():
    tpl = get_template("tpl-single-qa")
    assert tpl is not None
    assert tpl["id"] == "tpl-single-qa"


def test_get_template_not_found():
    assert get_template("no-such-template") is None


def test_all_templates_have_valid_case_ids():
    for tpl in list_templates():
        for case in tpl["cases"]:
            assert "id" in case
            assert "type" in case


def test_template_eval_rules_reference_valid_types():
    valid_types = {"keyword_match", "response_time", "llm_score", "semantic_similarity", "json_schema", "safety"}
    for tpl in list_templates():
        for case in tpl["cases"]:
            for rule in case.get("eval_rules", []):
                assert rule["type"] in valid_types, f"Unknown rule type {rule['type']} in template {tpl['id']}"


# ── scenario loader ───────────────────────────────────────────────────────

def _write_tmp(content: str, suffix: str) -> Path:
    f = tempfile.NamedTemporaryFile(mode="w", suffix=suffix, delete=False, encoding="utf-8")
    f.write(content)
    f.flush()
    return Path(f.name)


def test_load_json_scenario():
    data = {
        "name": "JSON 场景",
        "cases": [{"id": "c1", "type": "single", "messages": ["你好"]}],
    }
    path = _write_tmp(json.dumps(data), ".json")
    scenario = load_scenario_file(path)
    assert scenario.name == "JSON 场景"
    assert len(scenario.cases) == 1
    path.unlink()


def test_load_yaml_scenario():
    data = {
        "name": "YAML 场景",
        "cases": [{"id": "c1", "type": "single", "messages": ["Hello"]}],
    }
    path = _write_tmp(yaml.dump(data, allow_unicode=True), ".yaml")
    scenario = load_scenario_file(path)
    assert scenario.name == "YAML 场景"
    path.unlink()


def test_load_yml_extension():
    data = {
        "name": "YML 场景",
        "cases": [{"id": "c1", "type": "single", "messages": ["test"]}],
    }
    path = _write_tmp(yaml.dump(data, allow_unicode=True), ".yml")
    scenario = load_scenario_file(path)
    assert scenario.name == "YML 场景"
    path.unlink()


def test_load_scenario_with_tags():
    data = {
        "name": "带标签",
        "tags": ["health", "basic"],
        "cases": [{"id": "c1", "type": "single", "messages": ["hi"]}],
    }
    path = _write_tmp(json.dumps(data), ".json")
    scenario = load_scenario_file(path)
    assert "health" in scenario.tags
    path.unlink()


def test_load_scenario_with_eval_rules():
    data = {
        "name": "含规则",
        "cases": [{
            "id": "c1", "type": "single", "messages": ["hi"],
            "eval_rules": [{"type": "response_time", "params": {"max_ms": 5000}}],
        }],
    }
    path = _write_tmp(json.dumps(data), ".json")
    scenario = load_scenario_file(path)
    assert len(scenario.cases[0].eval_rules) == 1
    assert scenario.cases[0].eval_rules[0].type == "response_time"
    path.unlink()


def test_load_scenario_file_not_found():
    with pytest.raises(ScenarioValidationError, match="not found"):
        load_scenario_file(Path("/tmp/nonexistent-scenario-xyz.json"))


def test_load_unsupported_extension():
    path = Path(_write_tmp("{}", ".txt"))
    with pytest.raises(ScenarioValidationError, match="unsupported"):
        load_scenario_file(path)
    path.unlink()


def test_validate_scenario_valid():
    data = {
        "name": "有效场景",
        "cases": [{"id": "c1", "type": "single", "messages": ["hi"]}],
    }
    valid, errors = validate_scenario(data)
    assert valid is True
    assert errors == []


def test_validate_scenario_missing_name():
    data = {"cases": [{"id": "c1", "type": "single", "messages": ["hi"]}]}
    valid, errors = validate_scenario(data)
    assert valid is False
    assert len(errors) > 0


def test_validate_scenario_empty_cases():
    data = {"name": "空用例", "cases": []}  # empty list triggers the cases_not_empty validator
    valid, errors = validate_scenario(data)
    assert valid is False