"""Unit tests for scenarios loader and templates.""" import json import tempfile from pathlib import Path import pytest import yaml from agenteval.scenarios.loader import load_scenario_file, validate_scenario, ScenarioValidationError from agenteval.scenarios.templates import get_template, list_templates # ── templates ───────────────────────────────────────────────────────────── def test_list_templates_returns_list(): tpls = list_templates() assert isinstance(tpls, list) assert len(tpls) >= 6 def test_template_has_required_fields(): for tpl in list_templates(): assert "id" in tpl assert "name" in tpl assert "description" in tpl assert "cases" in tpl assert isinstance(tpl["cases"], list) assert len(tpl["cases"]) > 0 def test_get_template_by_id(): tpl = get_template("tpl-single-qa") assert tpl is not None assert tpl["id"] == "tpl-single-qa" def test_get_template_not_found(): assert get_template("no-such-template") is None def test_all_templates_have_valid_case_ids(): for tpl in list_templates(): for case in tpl["cases"]: assert "id" in case assert "type" in case def test_template_eval_rules_reference_valid_types(): valid_types = {"keyword_match", "response_time", "llm_score", "semantic_similarity", "json_schema", "safety"} for tpl in list_templates(): for case in tpl["cases"]: for rule in case.get("eval_rules", []): assert rule["type"] in valid_types, f"Unknown rule type {rule['type']} in template {tpl['id']}" # ── scenario loader ─────────────────────────────────────────────────────── def _write_tmp(content: str, suffix: str) -> Path: f = tempfile.NamedTemporaryFile(mode="w", suffix=suffix, delete=False, encoding="utf-8") f.write(content) f.flush() return Path(f.name) def test_load_json_scenario(): data = { "name": "JSON 场景", "cases": [{"id": "c1", "type": "single", "messages": ["你好"]}], } path = _write_tmp(json.dumps(data), ".json") scenario = load_scenario_file(path) assert scenario.name == "JSON 场景" assert len(scenario.cases) == 1 path.unlink() def test_load_yaml_scenario(): data = { "name": "YAML 场景", "cases": [{"id": "c1", "type": "single", "messages": ["Hello"]}], } path = _write_tmp(yaml.dump(data, allow_unicode=True), ".yaml") scenario = load_scenario_file(path) assert scenario.name == "YAML 场景" path.unlink() def test_load_yml_extension(): data = { "name": "YML 场景", "cases": [{"id": "c1", "type": "single", "messages": ["test"]}], } path = _write_tmp(yaml.dump(data, allow_unicode=True), ".yml") scenario = load_scenario_file(path) assert scenario.name == "YML 场景" path.unlink() def test_load_scenario_with_tags(): data = { "name": "带标签", "tags": ["health", "basic"], "cases": [{"id": "c1", "type": "single", "messages": ["hi"]}], } path = _write_tmp(json.dumps(data), ".json") scenario = load_scenario_file(path) assert "health" in scenario.tags path.unlink() def test_load_scenario_with_eval_rules(): data = { "name": "含规则", "cases": [{ "id": "c1", "type": "single", "messages": ["hi"], "eval_rules": [{"type": "response_time", "params": {"max_ms": 5000}}], }], } path = _write_tmp(json.dumps(data), ".json") scenario = load_scenario_file(path) assert len(scenario.cases[0].eval_rules) == 1 assert scenario.cases[0].eval_rules[0].type == "response_time" path.unlink() def test_load_scenario_file_not_found(): with pytest.raises(ScenarioValidationError, match="not found"): load_scenario_file(Path("/tmp/nonexistent-scenario-xyz.json")) def test_load_unsupported_extension(): path = Path(_write_tmp("{}", ".txt")) with pytest.raises(ScenarioValidationError, match="unsupported"): load_scenario_file(path) path.unlink() def test_validate_scenario_valid(): data = { "name": "有效场景", "cases": [{"id": "c1", "type": "single", "messages": ["hi"]}], } valid, errors = validate_scenario(data) assert valid is True assert errors == [] def test_validate_scenario_missing_name(): data = {"cases": [{"id": "c1", "type": "single", "messages": ["hi"]}]} valid, errors = validate_scenario(data) assert valid is False assert len(errors) > 0 def test_validate_scenario_empty_cases(): data = {"name": "空用例", "cases": []} # empty list triggers the cases_not_empty validator valid, errors = validate_scenario(data) assert valid is False