diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index c9b37e8..bb62884 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -7,7 +7,7 @@ from typing import Any, Optional from jinja2 import Template -from agenteval.storage.db import DATA_DIR +from agenteval.storage.db import DATA_DIR, iso_utc from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository from agenteval.utils.llm import extract_reply_text @@ -154,8 +154,8 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: "scenario_id": run.scenario_id, "scenario_name": scenario.name if scenario else "未知", "status": run.status.value, - "started_at": run.started_at.isoformat() if run.started_at else None, - "completed_at": run.completed_at.isoformat() if run.completed_at else None, + "started_at": iso_utc(run.started_at), + "completed_at": iso_utc(run.completed_at), "summary": { "total_cases": summary.get("total_cases", 0), "passed_cases": summary.get("passed_cases", 0), @@ -191,14 +191,16 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[ return None return all(r["passed"] for r in c.get("results", [])) - case_diffs.append({ - "case_id": cid, - "run_a_passed": _case_passed(ca), - "run_b_passed": _case_passed(cb), - "changed": _case_passed(ca) != _case_passed(cb), - "run_a_results": ca["results"] if ca else [], - "run_b_results": cb["results"] if cb else [], - }) + case_diffs.append( + { + "case_id": cid, + "run_a_passed": _case_passed(ca), + "run_b_passed": _case_passed(cb), + "changed": _case_passed(ca) != _case_passed(cb), + "run_a_results": ca["results"] if ca else [], + "run_b_results": cb["results"] if cb else [], + } + ) return { "run_a": { diff --git a/backend/agenteval/storage/db.py b/backend/agenteval/storage/db.py index 98049a7..e33458f 100644 --- a/backend/agenteval/storage/db.py +++ b/backend/agenteval/storage/db.py @@ -27,6 +27,19 @@ def utc_now() -> datetime: return datetime.now(timezone.utc) +def iso_utc(dt: datetime | None) -> str | None: + """Serialize a datetime to ISO 8601 with UTC timezone suffix. + + Guarantees the output always ends with 'Z' or '+00:00' so JavaScript's + Date.parse() interprets it correctly as UTC (no 8-hour local-time offset). + """ + if dt is None: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.isoformat(timespec="seconds").replace("+00:00", "Z") + + def new_uuid() -> str: return str(uuid.uuid4()) diff --git a/backend/agenteval/web/routers/files.py b/backend/agenteval/web/routers/files.py index 82c3cb6..af0cdbe 100644 --- a/backend/agenteval/web/routers/files.py +++ b/backend/agenteval/web/routers/files.py @@ -10,7 +10,7 @@ from pydantic import BaseModel from sqlmodel import Session from agenteval.config import get_settings -from agenteval.storage.db import FILES_DIR +from agenteval.storage.db import FILES_DIR, iso_utc from agenteval.storage.file_repository import FileCategoryRepository, FileRecordRepository from agenteval.web.deps import get_db @@ -112,7 +112,7 @@ def list_files(category_id: str | None = None, session: Session = Depends(get_db "mime_type": r.mime_type, "file_ext": r.file_ext, "category_id": r.category_id, - "created_at": r.created_at.isoformat() if r.created_at else None, + "created_at": iso_utc(r.created_at), } for r in records ] @@ -174,7 +174,7 @@ async def upload_file( "mime_type": record.mime_type, "file_ext": record.file_ext, "category_id": record.category_id, - "created_at": record.created_at.isoformat() if record.created_at else None, + "created_at": iso_utc(record.created_at), } diff --git a/backend/agenteval/web/routers/runs.py b/backend/agenteval/web/routers/runs.py index 4a65123..5c1181c 100644 --- a/backend/agenteval/web/routers/runs.py +++ b/backend/agenteval/web/routers/runs.py @@ -9,7 +9,7 @@ from sqlmodel import Session from agenteval.evaluation.engine import EvalEngine from agenteval.models import EvalRun, RunStatus -from agenteval.storage.db import get_session +from agenteval.storage.db import get_session, iso_utc from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository from agenteval.utils.llm import extract_reply_text from agenteval.utils.webhook import send_run_webhook @@ -148,8 +148,8 @@ async def get_run_logs(run_id: str, session: Session = Depends(get_db)) -> dict: "latency_ms": t.latency_ms, "sent_text": t.get_sent_message().get("msgBody", {}).get("content", ""), "reply_text": extract_reply_text(t.get_reply()), - "sent_at": t.sent_at.isoformat() if t.sent_at else None, - "received_at": t.received_at.isoformat() if t.received_at else None, + "sent_at": iso_utc(t.sent_at), + "received_at": iso_utc(t.received_at), } for t in turns ] diff --git a/backend/plugins/openclaw/agenteval_skill.py b/backend/plugins/openclaw/agenteval_skill.py index ab89548..ba6048b 100644 --- a/backend/plugins/openclaw/agenteval_skill.py +++ b/backend/plugins/openclaw/agenteval_skill.py @@ -39,10 +39,13 @@ class AgentEvalSkill: def run(self) -> dict[str, Any]: with httpx.Client(base_url=self.api_base, headers=self._headers(), timeout=30) as client: # 1. Start the run - resp = client.post("/api/runs", json={ - "target_id": self.target_id, - "scenario_id": self.scenario_id, - }) + resp = client.post( + "/api/runs", + json={ + "target_id": self.target_id, + "scenario_id": self.scenario_id, + }, + ) if resp.status_code != 200: return {"ok": False, "error": f"启动评测失败: HTTP {resp.status_code} {resp.text}"} diff --git a/tests/integration/test_reports_api.py b/tests/integration/test_reports_api.py new file mode 100644 index 0000000..dc06a87 --- /dev/null +++ b/tests/integration/test_reports_api.py @@ -0,0 +1,160 @@ +"""Integration tests for the reports API: /api/reports/{id}, /markdown, /compare.""" + +import json +import pytest +from fastapi.testclient import TestClient +from sqlmodel import Session, SQLModel, create_engine + +from agenteval.models import ( + Case, CaseType, EvalResult, EvalRun, EvalTarget, RunStatus, Scenario, + PlatformType, ChannelType, TargetStatus, Turn, +) +from agenteval.storage.repository import ResultRepository, RunRepository, ScenarioRepository, TargetRepository +from agenteval.web.app import app +from agenteval.web.deps import get_db + + +@pytest.fixture() +def client_with_db(tmp_path): + from agenteval.storage.db import ( # noqa: F401 + EvalResultDB, EvalRunDB, EvalTargetDB, FileCategoryDB, FileRecordDB, ScenarioDB, TurnDB, + ) + engine = create_engine( + f"sqlite:///{tmp_path / 'reports_api.db'}", + connect_args={"check_same_thread": False}, + ) + SQLModel.metadata.create_all(engine) + session = Session(engine) + + def override_get_db(): + try: + yield session + finally: + pass + + app.dependency_overrides[get_db] = override_get_db + client = TestClient(app) + yield client, session + app.dependency_overrides.clear() + session.close() + + +def _seed_run(session: Session, name_suffix: str = "") -> str: + target = EvalTarget( + name=f"target{name_suffix}", + platform=PlatformType.AI_DIGITAL_EMPLOYEE, + channel_type=ChannelType.TUTU_API, + channel_config={}, + status=TargetStatus.ACTIVE, + ) + target = TargetRepository(session).create(target) + + scenario = Scenario( + name=f"scenario{name_suffix}", + cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])], + ) + scenario = ScenarioRepository(session).create(scenario) + + run = EvalRun( + target_id=target.id, + scenario_id=scenario.id, + status=RunStatus.COMPLETED, + ) + run = RunRepository(session).create(run) + + result_repo = ResultRepository(session) + run_repo = RunRepository(session) + + turn = Turn( + run_id=run.id, case_id="c1", round_index=1, + sent_message={"msgBody": {"content": "问题"}}, + reply={"msgBody": {"content": "回答"}}, + latency_ms=300, + ) + result_repo.save_turn(turn) + db_turn = run_repo.get_turns(run.id)[0] + + result_repo.save_result(EvalResult( + run_id=run.id, case_id="c1", + turn_id=db_turn.id or "", + rule_type="response_time", + passed=True, score=0.9, reason="快", + )) + + run.summary = { + "total_cases": 1, "passed_cases": 1, "failed_cases": 0, + "total_rules": 1, "passed_rules": 1, "pass_rate": 1.0, + } + RunRepository(session).update(run) + return run.id + + +def test_get_report_200(client_with_db): + client, session = client_with_db + run_id = _seed_run(session) + resp = client.get(f"/api/reports/{run_id}") + assert resp.status_code == 200 + data = resp.json() + assert data["run_id"] == run_id + assert data["summary"]["pass_rate"] == 1.0 + + +def test_get_report_404(client_with_db): + client, _ = client_with_db + resp = client.get("/api/reports/no-such-run") + assert resp.status_code == 404 + + +def test_get_html_report(client_with_db): + client, session = client_with_db + run_id = _seed_run(session) + resp = client.get(f"/api/reports/{run_id}/html") + assert resp.status_code == 200 + assert "text/html" in resp.headers["content-type"] + assert "评测报告" in resp.text + + +def test_get_json_report(client_with_db): + client, session = client_with_db + run_id = _seed_run(session) + resp = client.get(f"/api/reports/{run_id}/json") + assert resp.status_code == 200 + data = json.loads(resp.text) + assert "run_id" in data + + +def test_get_markdown_report(client_with_db): + client, session = client_with_db + run_id = _seed_run(session) + resp = client.get(f"/api/reports/{run_id}/markdown") + assert resp.status_code == 200 + assert "text/markdown" in resp.headers["content-type"] + assert "# 评测报告" in resp.text + assert "## 汇总" in resp.text + + +def test_get_markdown_report_attachment_header(client_with_db): + client, session = client_with_db + run_id = _seed_run(session) + resp = client.get(f"/api/reports/{run_id}/markdown") + assert "attachment" in resp.headers.get("content-disposition", "") + + +def test_compare_report(client_with_db): + client, session = client_with_db + run_id_a = _seed_run(session, "A") + run_id_b = _seed_run(session, "B") + resp = client.get(f"/api/reports/compare?run1={run_id_a}&run2={run_id_b}") + assert resp.status_code == 200 + data = resp.json() + assert data["run_a"]["run_id"] == run_id_a + assert data["run_b"]["run_id"] == run_id_b + assert "delta" in data + assert "cases" in data + + +def test_compare_report_run_not_found(client_with_db): + client, session = client_with_db + run_id = _seed_run(session) + resp = client.get(f"/api/reports/compare?run1={run_id}&run2=ghost-id") + assert resp.status_code == 404 diff --git a/tests/unit/test_file_repository.py b/tests/unit/test_file_repository.py new file mode 100644 index 0000000..2da7a23 --- /dev/null +++ b/tests/unit/test_file_repository.py @@ -0,0 +1,220 @@ +"""Unit tests for FileCategoryRepository and FileRecordRepository.""" + +import uuid +from pathlib import Path + +import pytest +from sqlmodel import Session, SQLModel, create_engine + +from agenteval.storage.file_repository import FileCategoryRepository, FileRecordRepository + + +@pytest.fixture() +def file_db_session(tmp_path: Path): + """Isolated SQLite session with file management tables.""" + from agenteval.storage.db import ( # noqa: F401 + EvalResultDB, EvalRunDB, EvalTargetDB, FileCategoryDB, FileRecordDB, ScenarioDB, TurnDB, + ) + engine = create_engine( + f"sqlite:///{tmp_path / 'file_test.db'}", + connect_args={"check_same_thread": False}, + ) + SQLModel.metadata.create_all(engine) + session = Session(engine) + try: + yield session + finally: + session.close() + engine.dispose() + + +@pytest.fixture() +def fake_files_dir(tmp_path: Path, monkeypatch): + """Redirect FILES_DIR to a temp directory so tests don't touch data/files.""" + import agenteval.storage.db as db_mod + import agenteval.storage.file_repository as repo_mod + + fake_dir = tmp_path / "files" + fake_dir.mkdir() + monkeypatch.setattr(db_mod, "FILES_DIR", fake_dir) + monkeypatch.setattr(repo_mod, "FILES_DIR", fake_dir) + return fake_dir + + +# ── FileCategoryRepository ──────────────────────────────────────────────── + +def test_create_root_category(file_db_session): + repo = FileCategoryRepository(file_db_session) + cat = repo.create("文件一类") + assert cat.id is not None + assert cat.name == "文件一类" + assert cat.parent_id is None + + +def test_create_child_category(file_db_session): + repo = FileCategoryRepository(file_db_session) + root = repo.create("根分类") + child = repo.create("子分类", parent_id=root.id) + assert child.parent_id == root.id + + +def test_list_all_categories(file_db_session): + repo = FileCategoryRepository(file_db_session) + repo.create("A") + repo.create("B") + all_cats = repo.list_all() + assert len(all_cats) == 2 + + +def test_get_tree_structure(file_db_session): + repo = FileCategoryRepository(file_db_session) + root = repo.create("root") + repo.create("child1", parent_id=root.id) + repo.create("child2", parent_id=root.id) + tree = repo.get_tree() + assert len(tree) == 1 + assert tree[0]["key"] == root.id + assert len(tree[0]["children"]) == 2 + + +def test_get_tree_flat(file_db_session): + repo = FileCategoryRepository(file_db_session) + repo.create("A") + repo.create("B") + tree = repo.get_tree() + assert len(tree) == 2 + assert all(len(node["children"]) == 0 for node in tree) + + +def test_update_category_name(file_db_session): + repo = FileCategoryRepository(file_db_session) + cat = repo.create("旧名称") + updated = repo.update(cat.id, "新名称") + assert updated is not None + assert updated.name == "新名称" + + +def test_update_nonexistent_returns_none(file_db_session): + repo = FileCategoryRepository(file_db_session) + assert repo.update("nonexistent-id", "name") is None + + +def test_get_nonexistent_returns_none(file_db_session): + repo = FileCategoryRepository(file_db_session) + assert repo.get("no-such-id") is None + + +def test_delete_category(file_db_session): + repo = FileCategoryRepository(file_db_session) + cat = repo.create("删除目标") + assert repo.delete(cat.id) is True + assert repo.get(cat.id) is None + + +def test_delete_nonexistent_returns_false(file_db_session): + repo = FileCategoryRepository(file_db_session) + assert repo.delete("ghost-id") is False + + +def test_delete_cascades_to_children(file_db_session, fake_files_dir): + cat_repo = FileCategoryRepository(file_db_session) + file_repo = FileRecordRepository(file_db_session) + + root = cat_repo.create("父") + child = cat_repo.create("子", parent_id=root.id) + + # Create a file in child + file_repo.create("f.txt", "f-storage.txt", 10, "text/plain", "txt", category_id=child.id) + + assert cat_repo.delete(root.id) is True + assert cat_repo.get(root.id) is None + assert cat_repo.get(child.id) is None + assert len(file_repo.list_all()) == 0 + + +# ── FileRecordRepository ────────────────────────────────────────────────── + +def test_create_file_record(file_db_session): + repo = FileRecordRepository(file_db_session) + rec = repo.create("test.txt", "stored-uuid.txt", 128, "text/plain", "txt") + assert rec.id is not None + assert rec.original_name == "test.txt" + assert rec.file_size == 128 + + +def test_create_file_with_category(file_db_session): + cat_repo = FileCategoryRepository(file_db_session) + file_repo = FileRecordRepository(file_db_session) + cat = cat_repo.create("分类") + rec = file_repo.create("data.json", "stored.json", 256, "application/json", "json", category_id=cat.id) + assert rec.category_id == cat.id + + +def test_list_files_all(file_db_session): + repo = FileRecordRepository(file_db_session) + repo.create("a.txt", "a-s.txt", 1, "text/plain", "txt") + repo.create("b.txt", "b-s.txt", 2, "text/plain", "txt") + assert len(repo.list_all()) == 2 + + +def test_list_files_by_category(file_db_session): + cat_repo = FileCategoryRepository(file_db_session) + file_repo = FileRecordRepository(file_db_session) + cat_a = cat_repo.create("A") + cat_b = cat_repo.create("B") + file_repo.create("in_a.txt", "s1.txt", 1, "text/plain", "txt", category_id=cat_a.id) + file_repo.create("in_b.txt", "s2.txt", 2, "text/plain", "txt", category_id=cat_b.id) + file_repo.create("no_cat.txt", "s3.txt", 3, "text/plain", "txt") + + only_a = file_repo.list_all(category_id=cat_a.id) + assert len(only_a) == 1 + assert only_a[0].original_name == "in_a.txt" + + +def test_list_files_includes_subcategory_files(file_db_session): + cat_repo = FileCategoryRepository(file_db_session) + file_repo = FileRecordRepository(file_db_session) + root = cat_repo.create("root") + child = cat_repo.create("child", parent_id=root.id) + file_repo.create("in_root.txt", "sr.txt", 1, "text/plain", "txt", category_id=root.id) + file_repo.create("in_child.txt", "sc.txt", 2, "text/plain", "txt", category_id=child.id) + + files = file_repo.list_all(category_id=root.id) + assert len(files) == 2 + + +def test_get_file_record(file_db_session): + repo = FileRecordRepository(file_db_session) + rec = repo.create("get_me.txt", "gm.txt", 10, "text/plain", "txt") + fetched = repo.get(rec.id) + assert fetched is not None + assert fetched.id == rec.id + + +def test_get_nonexistent_file_returns_none(file_db_session): + repo = FileRecordRepository(file_db_session) + assert repo.get("no-such-id") is None + + +def test_delete_file_record(file_db_session): + repo = FileRecordRepository(file_db_session) + rec = repo.create("del.txt", "del-s.txt", 5, "text/plain", "txt") + assert repo.delete(rec.id) is True + assert repo.get(rec.id) is None + + +def test_delete_removes_physical_file(file_db_session, fake_files_dir): + repo = FileRecordRepository(file_db_session) + storage_name = f"{uuid.uuid4()}.txt" + physical = fake_files_dir / storage_name + physical.write_text("content") + assert physical.exists() + + rec = repo.create("orig.txt", storage_name, 7, "text/plain", "txt") + repo.delete(rec.id) + assert not physical.exists() + + +def test_delete_nonexistent_file_returns_false(file_db_session): + repo = FileRecordRepository(file_db_session) + assert repo.delete("ghost-id") is False diff --git a/tests/unit/test_llm_score.py b/tests/unit/test_llm_score.py new file mode 100644 index 0000000..f3a6cbe --- /dev/null +++ b/tests/unit/test_llm_score.py @@ -0,0 +1,191 @@ +"""Unit tests for LlmScoreRule — mocking httpx to avoid real API calls.""" + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from agenteval.evaluation.rules.llm_score import LlmScoreRule +from agenteval.models import Case, CaseType, Turn + + +def _turn(reply_text: str, sent_text: str = "问题", latency_ms: int = 500) -> Turn: + return Turn( + id="t1", run_id="r1", case_id="c1", round_index=1, + sent_message={"msgBody": {"content": sent_text}}, + reply={"msgBody": {"content": reply_text}}, + latency_ms=latency_ms, + ) + + +def _case() -> Case: + return Case(id="c1", type=CaseType.SINGLE, messages=["hi"]) + + +def _make_llm_response(score: float, reason: str = "ok") -> MagicMock: + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.json = MagicMock(return_value={ + "choices": [{"message": {"content": json.dumps({"score": score, "reason": reason})}}] + }) + return mock_resp + + +def _make_content_block_response(score: float, reason: str = "ok") -> MagicMock: + """Simulate Anthropic content-block-array format.""" + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.json = MagicMock(return_value={ + "choices": [{ + "message": { + "content": [ + {"type": "text", "text": json.dumps({"score": score, "reason": reason})} + ] + } + }] + }) + return mock_resp + + +# ── basic evaluate ──────────────────────────────────────────────────────── + +async def test_llm_score_no_api_url_fails(): + rule = LlmScoreRule({"criteria": "礼貌"}) + result = await rule.evaluate(_case(), [_turn("回答内容")]) + assert result.passed is False + assert "api_url" in result.reason + + +async def test_llm_score_empty_dialog_fails(): + rule = LlmScoreRule({"api_url": "http://mock", "min_score": 7}) + result = await rule.evaluate(_case(), []) + assert result.passed is False + assert "无回复" in result.reason + + +# ── OpenAI format ───────────────────────────────────────────────────────── + +async def test_llm_score_passes_above_threshold(): + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6}) + mock_resp = _make_llm_response(score=8.0, reason="很好") + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(return_value=mock_resp) + result = await rule.evaluate(_case(), [_turn("优质回答")]) + + assert result.passed is True + assert result.score == pytest.approx(0.8) + assert "8" in result.reason + + +async def test_llm_score_fails_below_threshold(): + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 7}) + mock_resp = _make_llm_response(score=4.0, reason="较差") + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(return_value=mock_resp) + result = await rule.evaluate(_case(), [_turn("差劲回答")]) + + assert result.passed is False + assert result.score == pytest.approx(0.4) + + +async def test_llm_score_clamps_score_to_0_10(): + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5}) + # API returns out-of-range score + mock_resp = _make_llm_response(score=12.0) + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(return_value=mock_resp) + result = await rule.evaluate(_case(), [_turn("answer")]) + + assert result.score == pytest.approx(1.0) # clamped 10/10 = 1.0 + + +# ── Anthropic content-block format ──────────────────────────────────────── + +async def test_llm_score_handles_content_block_array(): + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6}) + mock_resp = _make_content_block_response(score=7.5) + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(return_value=mock_resp) + result = await rule.evaluate(_case(), [_turn("answer")]) + + assert result.passed is True + assert result.score == pytest.approx(0.75) + + +# ── JSON fallback parsing ───────────────────────────────────────────────── + +async def test_llm_score_parses_json_with_preamble(): + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6}) + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.json = MagicMock(return_value={ + "choices": [{"message": {"content": 'Sure! Here is the result: {"score": 7, "reason": "decent"}'}}] + }) + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(return_value=mock_resp) + result = await rule.evaluate(_case(), [_turn("answer")]) + + assert result.passed is True + + +# ── error handling ──────────────────────────────────────────────────────── + +async def test_llm_score_api_error_fails_gracefully(): + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6}) + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(side_effect=Exception("connection timeout")) + result = await rule.evaluate(_case(), [_turn("answer")]) + + assert result.passed is False + assert "timeout" in result.reason.lower() or "LLM" in result.reason + + +async def test_llm_score_empty_content_fails(): + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 6}) + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.json = MagicMock(return_value={"choices": []}) + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(return_value=mock_resp) + result = await rule.evaluate(_case(), [_turn("answer")]) + + assert result.passed is False + + +# ── question extraction ─────────────────────────────────────────────────── + +async def test_llm_score_extracts_question_from_sent_message(): + """Verifies that sent_message is used as the question when dialog has 1 turn.""" + rule = LlmScoreRule({"api_url": "http://mock/v1/chat", "min_score": 5}) + captured_payload = {} + + mock_resp = _make_llm_response(score=8.0) + + async def capture_post(url, **kwargs): + captured_payload.update(kwargs.get("json", {})) + return mock_resp + + with patch("agenteval.evaluation.rules.llm_score.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(side_effect=capture_post) + await rule.evaluate(_case(), [_turn("答案内容", sent_text="这是用户的问题")]) + + # The user_prompt should contain the sent question text + messages = captured_payload.get("messages", []) + user_msg = next((m for m in messages if m["role"] == "user"), None) + assert user_msg is not None + assert "这是用户的问题" in user_msg["content"] diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py new file mode 100644 index 0000000..dcc93c3 --- /dev/null +++ b/tests/unit/test_report.py @@ -0,0 +1,227 @@ +"""Unit tests for report generation: generate_report, generate_compare_report, render_markdown_report.""" + +import pytest +from sqlmodel import Session, SQLModel, create_engine + +from agenteval.evaluation.report import ( + generate_compare_report, + generate_report, + render_markdown_report, + render_json_report, +) +from agenteval.models import ( + Case, CaseType, EvalResult, EvalRun, EvalTarget, RunStatus, Scenario, Turn, + PlatformType, ChannelType, TargetStatus, +) +from agenteval.storage.repository import ResultRepository, RunRepository, ScenarioRepository, TargetRepository + + +@pytest.fixture() +def report_session(tmp_path): + from agenteval.storage.db import ( # noqa: F401 + EvalResultDB, EvalRunDB, EvalTargetDB, FileCategoryDB, FileRecordDB, ScenarioDB, TurnDB, + ) + engine = create_engine( + f"sqlite:///{tmp_path / 'report_test.db'}", + connect_args={"check_same_thread": False}, + ) + SQLModel.metadata.create_all(engine) + session = Session(engine) + try: + yield session + finally: + session.close() + engine.dispose() + + +def _seed_run(session: Session, *, pass_rate: float = 1.0, n_cases: int = 1) -> str: + """Create a minimal completed run with real data in the DB and return run_id.""" + target = EvalTarget( + name="测试对象", + platform=PlatformType.AI_DIGITAL_EMPLOYEE, + channel_type=ChannelType.TUTU_API, + channel_config={}, + status=TargetStatus.ACTIVE, + ) + target = TargetRepository(session).create(target) + + scenario = Scenario( + name="测试场景", + cases=[Case(id=f"c{i}", type=CaseType.SINGLE, messages=["hi"]) for i in range(n_cases)], + ) + scenario = ScenarioRepository(session).create(scenario) + + run = EvalRun( + target_id=target.id, + scenario_id=scenario.id, + status=RunStatus.COMPLETED, + ) + run = RunRepository(session).create(run) + + run_repo = RunRepository(session) + result_repo = ResultRepository(session) + + total = n_cases + passed = int(total * pass_rate) + + for i in range(n_cases): + turn = Turn( + run_id=run.id, + case_id=f"c{i}", + round_index=1, + sent_message={"msgBody": {"content": f"问题{i}"}}, + reply={"msgBody": {"content": f"回答{i}"}}, + latency_ms=200, + ) + result_repo.save_turn(turn) + db_turn = run_repo.get_turns(run.id)[-1] + + eval_result = EvalResult( + run_id=run.id, + case_id=f"c{i}", + turn_id=db_turn.id or "", + rule_type="keyword_match", + passed=(i < passed), + score=1.0 if i < passed else 0.0, + reason="通过" if i < passed else "失败", + ) + result_repo.save_result(eval_result) + + run.summary = { + "total_cases": total, + "passed_cases": passed, + "failed_cases": total - passed, + "total_rules": total, + "passed_rules": passed, + "pass_rate": round(pass_rate, 4), + } + RunRepository(session).update(run) + return run.id + + +# ── generate_report ─────────────────────────────────────────────────────── + +def test_generate_report_structure(report_session): + run_id = _seed_run(report_session) + report = generate_report(run_id, report_session) + + assert report["run_id"] == run_id + assert report["target_name"] == "测试对象" + assert report["scenario_name"] == "测试场景" + assert report["status"] == "completed" + assert "summary" in report + assert "cases" in report + + +def test_generate_report_summary_values(report_session): + run_id = _seed_run(report_session, pass_rate=1.0, n_cases=2) + report = generate_report(run_id, report_session) + s = report["summary"] + assert s["total_cases"] == 2 + assert s["passed_cases"] == 2 + assert s["pass_rate"] == 1.0 + + +def test_generate_report_partial_pass(report_session): + run_id = _seed_run(report_session, pass_rate=0.5, n_cases=2) + report = generate_report(run_id, report_session) + s = report["summary"] + assert s["passed_cases"] == 1 + assert s["failed_cases"] == 1 + + +def test_generate_report_cases_contain_turns_and_results(report_session): + run_id = _seed_run(report_session, n_cases=1) + report = generate_report(run_id, report_session) + assert len(report["cases"]) == 1 + case = report["cases"][0] + assert len(case["turns"]) == 1 + assert len(case["results"]) == 1 + assert case["turns"][0]["sent_text"] == "问题0" + assert case["turns"][0]["latency_ms"] == 200 + + +def test_generate_report_not_found_raises(report_session): + with pytest.raises(ValueError, match="run not found"): + generate_report("no-such-id", report_session) + + +# ── generate_compare_report ─────────────────────────────────────────────── + +def test_compare_report_structure(report_session): + run_id_a = _seed_run(report_session, pass_rate=1.0, n_cases=2) + run_id_b = _seed_run(report_session, pass_rate=0.5, n_cases=2) + result = generate_compare_report(run_id_a, run_id_b, report_session) + + assert "run_a" in result + assert "run_b" in result + assert "delta" in result + assert "cases" in result + assert result["run_a"]["run_id"] == run_id_a + assert result["run_b"]["run_id"] == run_id_b + + +def test_compare_report_delta(report_session): + run_id_a = _seed_run(report_session, pass_rate=0.5, n_cases=2) + run_id_b = _seed_run(report_session, pass_rate=1.0, n_cases=2) + result = generate_compare_report(run_id_a, run_id_b, report_session) + assert result["delta"]["pass_rate"] > 0 # B improved over A + + +def test_compare_report_changed_cases(report_session): + run_id_a = _seed_run(report_session, pass_rate=1.0, n_cases=2) + run_id_b = _seed_run(report_session, pass_rate=0.5, n_cases=2) + result = generate_compare_report(run_id_a, run_id_b, report_session) + # At least one case changed (A all-pass vs B half-pass) + assert result["changed_cases"] >= 1 + + +def test_compare_report_case_level(report_session): + run_id_a = _seed_run(report_session, n_cases=1) + run_id_b = _seed_run(report_session, n_cases=1) + result = generate_compare_report(run_id_a, run_id_b, report_session) + assert len(result["cases"]) >= 1 + case = result["cases"][0] + assert "run_a_passed" in case + assert "run_b_passed" in case + assert "changed" in case + + +# ── render_markdown_report ──────────────────────────────────────────────── + +def test_render_markdown_contains_header(report_session): + run_id = _seed_run(report_session) + md = render_markdown_report(run_id, report_session) + assert "# 评测报告" in md + + +def test_render_markdown_contains_summary_table(report_session): + run_id = _seed_run(report_session) + md = render_markdown_report(run_id, report_session) + assert "## 汇总" in md + assert "| 指标 | 数值 |" in md + assert "通过率" in md + + +def test_render_markdown_contains_case_section(report_session): + run_id = _seed_run(report_session, n_cases=1) + md = render_markdown_report(run_id, report_session) + assert "## 用例明细" in md + assert "### " in md # case header + + +def test_render_markdown_contains_rule_table(report_session): + run_id = _seed_run(report_session) + md = render_markdown_report(run_id, report_session) + assert "**规则评估结果**" in md + assert "keyword_match" in md + + +# ── render_json_report ──────────────────────────────────────────────────── + +def test_render_json_report_is_valid_json(report_session): + import json + run_id = _seed_run(report_session) + json_text = render_json_report(run_id, report_session) + parsed = json.loads(json_text) + assert parsed["run_id"] == run_id diff --git a/tests/unit/test_scenarios.py b/tests/unit/test_scenarios.py new file mode 100644 index 0000000..8d2a0a7 --- /dev/null +++ b/tests/unit/test_scenarios.py @@ -0,0 +1,159 @@ +"""Unit tests for scenarios loader and templates.""" + +import json +import tempfile +from pathlib import Path + +import pytest +import yaml + +from agenteval.scenarios.loader import load_scenario_file, validate_scenario, ScenarioValidationError +from agenteval.scenarios.templates import get_template, list_templates + + +# ── templates ───────────────────────────────────────────────────────────── + +def test_list_templates_returns_list(): + tpls = list_templates() + assert isinstance(tpls, list) + assert len(tpls) >= 6 + + +def test_template_has_required_fields(): + for tpl in list_templates(): + assert "id" in tpl + assert "name" in tpl + assert "description" in tpl + assert "cases" in tpl + assert isinstance(tpl["cases"], list) + assert len(tpl["cases"]) > 0 + + +def test_get_template_by_id(): + tpl = get_template("tpl-single-qa") + assert tpl is not None + assert tpl["id"] == "tpl-single-qa" + + +def test_get_template_not_found(): + assert get_template("no-such-template") is None + + +def test_all_templates_have_valid_case_ids(): + for tpl in list_templates(): + for case in tpl["cases"]: + assert "id" in case + assert "type" in case + + +def test_template_eval_rules_reference_valid_types(): + valid_types = {"keyword_match", "response_time", "llm_score", "semantic_similarity", "json_schema", "safety"} + for tpl in list_templates(): + for case in tpl["cases"]: + for rule in case.get("eval_rules", []): + assert rule["type"] in valid_types, f"Unknown rule type {rule['type']} in template {tpl['id']}" + + +# ── scenario loader ─────────────────────────────────────────────────────── + +def _write_tmp(content: str, suffix: str) -> Path: + f = tempfile.NamedTemporaryFile(mode="w", suffix=suffix, delete=False, encoding="utf-8") + f.write(content) + f.flush() + return Path(f.name) + + +def test_load_json_scenario(): + data = { + "name": "JSON 场景", + "cases": [{"id": "c1", "type": "single", "messages": ["你好"]}], + } + path = _write_tmp(json.dumps(data), ".json") + scenario = load_scenario_file(path) + assert scenario.name == "JSON 场景" + assert len(scenario.cases) == 1 + path.unlink() + + +def test_load_yaml_scenario(): + data = { + "name": "YAML 场景", + "cases": [{"id": "c1", "type": "single", "messages": ["Hello"]}], + } + path = _write_tmp(yaml.dump(data, allow_unicode=True), ".yaml") + scenario = load_scenario_file(path) + assert scenario.name == "YAML 场景" + path.unlink() + + +def test_load_yml_extension(): + data = { + "name": "YML 场景", + "cases": [{"id": "c1", "type": "single", "messages": ["test"]}], + } + path = _write_tmp(yaml.dump(data, allow_unicode=True), ".yml") + scenario = load_scenario_file(path) + assert scenario.name == "YML 场景" + path.unlink() + + +def test_load_scenario_with_tags(): + data = { + "name": "带标签", + "tags": ["health", "basic"], + "cases": [{"id": "c1", "type": "single", "messages": ["hi"]}], + } + path = _write_tmp(json.dumps(data), ".json") + scenario = load_scenario_file(path) + assert "health" in scenario.tags + path.unlink() + + +def test_load_scenario_with_eval_rules(): + data = { + "name": "含规则", + "cases": [{ + "id": "c1", "type": "single", "messages": ["hi"], + "eval_rules": [{"type": "response_time", "params": {"max_ms": 5000}}], + }], + } + path = _write_tmp(json.dumps(data), ".json") + scenario = load_scenario_file(path) + assert len(scenario.cases[0].eval_rules) == 1 + assert scenario.cases[0].eval_rules[0].type == "response_time" + path.unlink() + + +def test_load_scenario_file_not_found(): + with pytest.raises(ScenarioValidationError, match="not found"): + load_scenario_file(Path("/tmp/nonexistent-scenario-xyz.json")) + + +def test_load_unsupported_extension(): + path = Path(_write_tmp("{}", ".txt")) + with pytest.raises(ScenarioValidationError, match="unsupported"): + load_scenario_file(path) + path.unlink() + + +def test_validate_scenario_valid(): + data = { + "name": "有效场景", + "cases": [{"id": "c1", "type": "single", "messages": ["hi"]}], + } + valid, errors = validate_scenario(data) + assert valid is True + assert errors == [] + + +def test_validate_scenario_missing_name(): + data = {"cases": [{"id": "c1", "type": "single", "messages": ["hi"]}]} + valid, errors = validate_scenario(data) + assert valid is False + assert len(errors) > 0 + + +def test_validate_scenario_empty_cases(): + data = {"name": "空用例", "cases": []} # empty list triggers the cases_not_empty validator + valid, errors = validate_scenario(data) + assert valid is False diff --git a/tests/unit/test_utils_llm.py b/tests/unit/test_utils_llm.py new file mode 100644 index 0000000..2b91ac7 --- /dev/null +++ b/tests/unit/test_utils_llm.py @@ -0,0 +1,149 @@ +"""Unit tests for agenteval.utils.llm — the shared LLM utility functions.""" + +import json +import pytest + +from agenteval.utils.llm import extract_reply_text, extract_content_from_llm_response, parse_json_from_llm_text + + +# ── extract_reply_text ─────────────────────────────────────────────────── + +def test_extract_reply_text_none(): + assert extract_reply_text(None) == "" + + +def test_extract_reply_text_str(): + assert extract_reply_text("hello") == "hello" + + +def test_extract_reply_text_dict_msgbody_dict(): + msg = {"msgBody": {"content": "inner text"}} + assert extract_reply_text(msg) == "inner text" + + +def test_extract_reply_text_dict_msgbody_str(): + msg = {"msgBody": "plain body"} + assert extract_reply_text(msg) == "plain body" + + +def test_extract_reply_text_dict_content_key(): + msg = {"content": "direct content"} + assert extract_reply_text(msg) == "direct content" + + +def test_extract_reply_text_dict_msgbody_none_fallback_content(): + # msgBody is falsy → falls back to content key + msg = {"msgBody": None, "content": "fallback"} + assert extract_reply_text(msg) == "fallback" + + +def test_extract_reply_text_int_coerces_to_str(): + assert extract_reply_text(42) == "42" + + +def test_extract_reply_text_empty_dict(): + assert extract_reply_text({}) == "" + + +# ── extract_content_from_llm_response ──────────────────────────────────── + +def test_extract_content_string_format(): + data = {"choices": [{"message": {"content": "answer text"}}]} + assert extract_content_from_llm_response(data) == "answer text" + + +def test_extract_content_block_array_text(): + data = { + "choices": [{ + "message": { + "content": [ + {"type": "text", "text": "block one"}, + {"type": "text", "text": "block two"}, + ] + } + }] + } + result = extract_content_from_llm_response(data) + assert "block one" in result + assert "block two" in result + + +def test_extract_content_block_array_skips_non_text(): + data = { + "choices": [{ + "message": { + "content": [ + {"type": "tool_use", "id": "t1", "input": {}}, + {"type": "text", "text": "real answer"}, + ] + } + }] + } + result = extract_content_from_llm_response(data) + assert result == "real answer" + + +def test_extract_content_block_uses_content_key_fallback(): + # Some providers use "content" instead of "text" inside blocks + data = { + "choices": [{ + "message": { + "content": [{"type": "text", "content": "via content key"}] + } + }] + } + assert extract_content_from_llm_response(data) == "via content key" + + +def test_extract_content_missing_choices(): + assert extract_content_from_llm_response({}) == "" + + +def test_extract_content_empty_choices(): + assert extract_content_from_llm_response({"choices": []}) == "" + + +def test_extract_content_integer_coerced(): + data = {"choices": [{"message": {"content": 123}}]} + assert extract_content_from_llm_response(data) == "123" + + +def test_extract_content_empty_block_list(): + data = {"choices": [{"message": {"content": []}}]} + assert extract_content_from_llm_response(data) == "" + + +# ── parse_json_from_llm_text ───────────────────────────────────────────── + +def test_parse_json_object(): + result = parse_json_from_llm_text('{"score": 8, "reason": "good"}') + assert result["score"] == 8 + + +def test_parse_json_array(): + result = parse_json_from_llm_text('["a", "b", "c"]') + assert result == ["a", "b", "c"] + + +def test_parse_json_with_surrounding_text(): + text = 'Here is the result: {"score": 7} and nothing else.' + result = parse_json_from_llm_text(text) + assert result["score"] == 7 + + +def test_parse_json_array_with_preamble(): + text = 'Generated questions: ["q1", "q2", "q3"]' + result = parse_json_from_llm_text(text) + assert result == ["q1", "q2", "q3"] + + +def test_parse_json_raises_on_no_json(): + with pytest.raises((ValueError, json.JSONDecodeError)): + parse_json_from_llm_text("no json here at all") + + +def test_parse_json_markdown_wrapped(): + text = '```json\n{"key": "value"}\n```' + # The fallback bracket-search finds the { in the markdown + result = parse_json_from_llm_text(text) + assert result["key"] == "value" diff --git a/tests/unit/test_webhook.py b/tests/unit/test_webhook.py new file mode 100644 index 0000000..fc8840f --- /dev/null +++ b/tests/unit/test_webhook.py @@ -0,0 +1,84 @@ +"""Unit tests for webhook notification utility.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + + +async def test_webhook_not_called_when_url_not_configured(): + with patch("agenteval.utils.webhook.get_settings") as mock_settings: + mock_settings.return_value.webhook_url = None + mock_settings.return_value.webhook_secret = None + + with patch("agenteval.utils.webhook.httpx.AsyncClient") as MockClient: + from agenteval.utils.webhook import send_run_webhook + await send_run_webhook("run-1", "completed", {"pass_rate": 1.0}) + MockClient.assert_not_called() + + +async def test_webhook_posts_correct_payload(): + captured = {} + + async def fake_post(url, *, json=None, headers=None, **kwargs): + captured["url"] = url + captured["json"] = json + captured["headers"] = headers + resp = MagicMock() + resp.status_code = 200 + return resp + + with patch("agenteval.utils.webhook.get_settings") as mock_settings: + mock_settings.return_value.webhook_url = "http://hook.example.com/notify" + mock_settings.return_value.webhook_secret = "secret123" + + with patch("agenteval.utils.webhook.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(side_effect=fake_post) + + from agenteval.utils.webhook import send_run_webhook + await send_run_webhook("run-42", "completed", {"pass_rate": 0.9}) + + assert captured["url"] == "http://hook.example.com/notify" + assert captured["json"]["run_id"] == "run-42" + assert captured["json"]["status"] == "completed" + assert captured["json"]["event"] == "run_completed" + assert captured["json"]["summary"]["pass_rate"] == 0.9 + assert captured["headers"]["X-Webhook-Secret"] == "secret123" + assert "/api/reports/run-42" in captured["json"]["report_url"] + + +async def test_webhook_no_secret_header(): + captured_headers = {} + + async def fake_post(url, *, json=None, headers=None, **kwargs): + captured_headers.update(headers or {}) + resp = MagicMock() + resp.status_code = 200 + return resp + + with patch("agenteval.utils.webhook.get_settings") as mock_settings: + mock_settings.return_value.webhook_url = "http://hook.example.com/notify" + mock_settings.return_value.webhook_secret = None + + with patch("agenteval.utils.webhook.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(side_effect=fake_post) + + from agenteval.utils.webhook import send_run_webhook + await send_run_webhook("run-1", "completed", {}) + + assert "X-Webhook-Secret" not in captured_headers + + +async def test_webhook_silently_ignores_errors(): + with patch("agenteval.utils.webhook.get_settings") as mock_settings: + mock_settings.return_value.webhook_url = "http://unreachable.example.com" + mock_settings.return_value.webhook_secret = None + + with patch("agenteval.utils.webhook.httpx.AsyncClient") as MockClient: + instance = MockClient.return_value.__aenter__.return_value + instance.post = AsyncMock(side_effect=Exception("connection refused")) + + from agenteval.utils.webhook import send_run_webhook + # Must not raise — errors are silently logged + await send_run_webhook("run-1", "failed", {})