From 739d586aecee0ceadbb7110bfe59561a35c8a7d8 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Tue, 28 Jul 2026 17:40:54 +0800 Subject: [PATCH] feat(backend): v0.4 triggered_by tracking, login gate, compare guard, dashboard stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - EvalRun.triggered_by 全链路（manual/ai_assistant/cli）+ 迁移 b7d4e6f81c22 - 标准 agenteval-run SKILL.md 纳入版本管理，deploy 脚本同步 + API Key 注入 - 简单登录：AGENTEVAL_ADMIN_PASSWORD + HMAC 会话 token，require_auth 双凭据 - 对比报告限同场景（400）+ 空 results 误判修复 - /api/stats/dashboard 扩展聚合；/api/runs 返回场景/对象名 - 测试 218 → 232 --- .env.example | 4 + backend/agenteval/config/settings.py | 7 ++ backend/agenteval/evaluation/engine.py | 4 + backend/agenteval/evaluation/report.py | 13 ++- backend/agenteval/models.py | 7 ++ backend/agenteval/storage/db.py | 1 + backend/agenteval/storage/repository.py | 2 + backend/agenteval/web/app.py | 4 +- backend/agenteval/web/deps.py | 40 +++++-- backend/agenteval/web/routers/auth.py | 52 +++++++++ backend/agenteval/web/routers/reports.py | 8 +- backend/agenteval/web/routers/runs.py | 20 +++- backend/agenteval/web/routers/stats.py | 63 ++++++++++- backend/cli/run.py | 3 +- backend/plugins/openclaw/README.md | 65 +++++------ backend/plugins/openclaw/agenteval_skill.py | 1 + .../openclaw/skills/agenteval-run/SKILL.md | 54 +++++++++ ...4e6f81c22_add_triggered_by_to_eval_runs.py | 29 +++++ pyproject.toml | 2 +- scripts/deploy-t480.sh | 17 +++ tests/integration/test_auth_api.py | 89 +++++++++++++++ tests/integration/test_reports_api.py | 28 +++-- tests/integration/test_runs_api.py | 42 +++++++ tests/integration/test_stats_api.py | 106 ++++++++++++++++++ tests/unit/test_report.py | 45 ++++++-- 25 files changed, 625 insertions(+), 81 deletions(-) create mode 100644 backend/agenteval/web/routers/auth.py create mode 100644 backend/plugins/openclaw/skills/agenteval-run/SKILL.md create mode 100644 migrations/versions/b7d4e6f81c22_add_triggered_by_to_eval_runs.py create mode 100644 tests/integration/test_auth_api.py create mode 100644 tests/integration/test_stats_api.py diff --git a/.env.example b/.env.example index 05b85db..bfbe7cb 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,10 @@ # In production, set a strong random token; clients must send it as X-API-Key. AGENTEVAL_API_KEY= +# If set, the web UI requires login with this password (session-level, stored +# in sessionStorage). Leave empty to disable the login gate (dev default). +AGENTEVAL_ADMIN_PASSWORD= + # Fernet key used to encrypt model API keys stored in SQLite. # Generate once with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" # Keep this value backed up with the database. Do not commit a real key. diff --git a/backend/agenteval/config/settings.py b/backend/agenteval/config/settings.py index 298227c..7aa9b7c 100644 --- a/backend/agenteval/config/settings.py +++ b/backend/agenteval/config/settings.py @@ -33,6 +33,13 @@ class Settings(BaseSettings): default=None, description="If set, all /api/* endpoints require the X-API-Key header to match.", ) + admin_password: Optional[str] = Field( + default=None, + description=( + "If set, the web UI requires login with this password. " + "Leave empty to disable the login gate (dev default)." + ), + ) secret_key: Optional[str] = Field( default=None, description="Fernet key used to encrypt model provider API keys at rest.", diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index f08b6e7..07b34ca 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -24,6 +24,7 @@ from agenteval.models import ( ModelPurpose, RuleLogic, RunStatus, + RunTrigger, Scenario, Turn, ) @@ -75,6 +76,7 @@ class EvalEngine: cancel_token: Optional[asyncio.Event] = None, timeout_config: Optional[TimeoutConfig] = None, max_concurrent_cases: int = 1, + triggered_by: RunTrigger = RunTrigger.MANUAL, ): self.target = target self.scenario = scenario @@ -84,6 +86,7 @@ class EvalEngine: self.result_repo = result_repo or ResultRepository(self.session) self.cancel_token = cancel_token or asyncio.Event() self.timeout_config = timeout_config or TimeoutConfig() + self.triggered_by = triggered_by self._case_semaphore = asyncio.Semaphore(max(1, max_concurrent_cases)) # Collects fatal case-level errors (e.g. dynamic message generation # failures) so their cause is persisted into run.summary — not just @@ -116,6 +119,7 @@ class EvalEngine: target_id=self.target.id or "", scenario_id=self.scenario.id or "", status=RunStatus.RUNNING, + triggered_by=self.triggered_by, started_at=utc_now(), ) run = self.run_repo.create(run) diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index bb62884..68a7bcd 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -169,10 +169,15 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[str, Any]: - """Build a side-by-side comparison dict for two runs.""" + """Build a side-by-side comparison dict for two runs of the same scenario.""" report_a = generate_report(run_id_1, session) report_b = generate_report(run_id_2, session) + # Cross-scenario case_ids never overlap, so every case would be flagged + # "changed" and the diff would be meaningless — reject early. + if report_a.get("scenario_id") != report_b.get("scenario_id"): + raise ValueError("compare report requires both runs to use the same scenario") + def _summary_delta(key: str) -> float: return report_b["summary"][key] - report_a["summary"][key] @@ -189,7 +194,11 @@ def generate_compare_report(run_id_1: str, run_id_2: str, session=None) -> dict[ def _case_passed(c): if not c: return None - return all(r["passed"] for r in c.get("results", [])) + results = c.get("results", []) + if not results: + # No rule results (e.g. errored case) must not count as passed. + return None + return all(r["passed"] for r in results) case_diffs.append( { diff --git a/backend/agenteval/models.py b/backend/agenteval/models.py index 4c008cb..bdc92b4 100644 --- a/backend/agenteval/models.py +++ b/backend/agenteval/models.py @@ -148,6 +148,12 @@ class RunStatus(str, Enum): FAILED = "failed" +class RunTrigger(str, Enum): + MANUAL = "manual" + AI_ASSISTANT = "ai_assistant" + CLI = "cli" + + class EvalRun(BaseModel): """A single evaluation run.""" @@ -155,6 +161,7 @@ class EvalRun(BaseModel): target_id: str scenario_id: str status: RunStatus = RunStatus.PENDING + triggered_by: RunTrigger = RunTrigger.MANUAL started_at: Optional[datetime] = None completed_at: Optional[datetime] = None summary: Optional[dict[str, Any]] = None diff --git a/backend/agenteval/storage/db.py b/backend/agenteval/storage/db.py index e8c921f..2997814 100644 --- a/backend/agenteval/storage/db.py +++ b/backend/agenteval/storage/db.py @@ -168,6 +168,7 @@ class EvalRunDB(SQLModel, table=True): target_id: Optional[str] = Field(default=None, foreign_key="eval_targets.id") scenario_id: Optional[str] = Field(default=None, foreign_key="scenarios.id") status: str = "pending" + triggered_by: str = Field(default="manual") started_at: Optional[datetime] = Field(default_factory=utc_now) completed_at: Optional[datetime] = None summary: Optional[str] = None diff --git a/backend/agenteval/storage/repository.py b/backend/agenteval/storage/repository.py index efdc101..819e8d6 100644 --- a/backend/agenteval/storage/repository.py +++ b/backend/agenteval/storage/repository.py @@ -82,6 +82,7 @@ def _run_to_db(run: EvalRun) -> EvalRunDB: target_id=run.target_id, scenario_id=run.scenario_id, status=run.status.value, + triggered_by=run.triggered_by.value, started_at=run.started_at, completed_at=run.completed_at, ) @@ -96,6 +97,7 @@ def _run_from_db(db: EvalRunDB) -> EvalRun: target_id=db.target_id, scenario_id=db.scenario_id, status=db.status, + triggered_by=db.triggered_by or "manual", started_at=db.started_at, completed_at=db.completed_at, summary=db.get_summary(), diff --git a/backend/agenteval/web/app.py b/backend/agenteval/web/app.py index 7cb195a..5435410 100644 --- a/backend/agenteval/web/app.py +++ b/backend/agenteval/web/app.py @@ -11,7 +11,7 @@ from agenteval.config import get_settings from agenteval.storage.db import init_db from agenteval.version import get_build_info, get_version from agenteval.web.deps import require_api_key -from agenteval.web.routers import files, model_configs, proxy, reports, runs, scenarios, stats, targets +from agenteval.web.routers import auth, files, model_configs, proxy, reports, runs, scenarios, stats, targets from agenteval.web.websocket import ws_manager @@ -40,6 +40,8 @@ app.add_middleware( _api_deps = [Depends(require_api_key)] +# Login endpoints must stay open — they are how the client obtains credentials. +app.include_router(auth.router, prefix="/api/auth", tags=["auth"]) app.include_router(targets.router, prefix="/api/targets", tags=["targets"], dependencies=_api_deps) app.include_router(scenarios.router, prefix="/api/scenarios", tags=["scenarios"], dependencies=_api_deps) app.include_router(runs.router, prefix="/api/runs", tags=["runs"], dependencies=_api_deps) diff --git a/backend/agenteval/web/deps.py b/backend/agenteval/web/deps.py index 84170dd..4b3859d 100644 --- a/backend/agenteval/web/deps.py +++ b/backend/agenteval/web/deps.py @@ -21,21 +21,37 @@ def get_db(): session.close() -def require_api_key(x_api_key: Optional[str] = Header(default=None)) -> None: - """Enforce X-API-Key header when AGENTEVAL_API_KEY is configured. +def require_auth( + x_api_key: Optional[str] = Header(default=None), + x_auth_token: Optional[str] = Header(default=None), +) -> None: + """Enforce authentication when any credential is configured. - When the setting is empty (default), the check is a no-op so local dev keeps - working without extra configuration. + Accepted credentials (either one passes): + - X-API-Key matching AGENTEVAL_API_KEY (machine callers, OpenClaw skill) + - X-Auth-Token matching the web session token (browser after login) + + When neither AGENTEVAL_API_KEY nor AGENTEVAL_ADMIN_PASSWORD is set, the + check is a no-op so local dev keeps working without extra configuration. """ - configured = get_settings().api_key - if not configured: + settings = get_settings() + if not settings.api_key and not settings.admin_password: return - if not x_api_key or x_api_key != configured: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid or missing X-API-Key", - ) + if settings.api_key and x_api_key == settings.api_key: + return + if settings.admin_password: + from agenteval.web.routers.auth import verify_session_token + if x_auth_token and verify_session_token(x_auth_token): + return + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or missing credentials", + ) + + +# Backwards-compatible alias: existing routers depend on require_api_key. +require_api_key = require_auth # Convenience alias used by routers via ``dependencies=[Depends(auth_required)]``. -auth_required = Depends(require_api_key) +auth_required = Depends(require_auth) diff --git a/backend/agenteval/web/routers/auth.py b/backend/agenteval/web/routers/auth.py new file mode 100644 index 0000000..f068e4c --- /dev/null +++ b/backend/agenteval/web/routers/auth.py @@ -0,0 +1,52 @@ +"""Session login for the web UI. + +A single shared password (AGENTEVAL_ADMIN_PASSWORD) gates the SPA. On success +the client receives a stateless HMAC token it sends as X-Auth-Token. Machine +callers (e.g. the OpenClaw skill) keep using X-API-Key and are unaffected. +""" + +import hmac +import secrets + +from fastapi import APIRouter, Header, HTTPException, status +from pydantic import BaseModel + +from agenteval.config import get_settings + +router = APIRouter() + +_TOKEN_CONTEXT = b"agenteval-web-session" + + +def session_token() -> str: + """Derive the stateless session token from configured secrets.""" + settings = get_settings() + key = (settings.secret_key or settings.admin_password or "").encode() + return hmac.new(key, _TOKEN_CONTEXT, "sha256").hexdigest() + + +def verify_session_token(token: str) -> bool: + return bool(token) and hmac.compare_digest(token, session_token()) + + +class LoginRequest(BaseModel): + password: str + + +@router.post("/login") +def login(request: LoginRequest) -> dict: + configured = get_settings().admin_password + if not configured: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="登录未启用") + if not secrets.compare_digest(request.password, configured): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="密码错误") + return {"token": session_token()} + + +@router.get("/status") +def auth_status(x_auth_token: str = Header(default="")) -> dict: + auth_required = bool(get_settings().admin_password) + return { + "auth_required": auth_required, + "authenticated": (not auth_required) or verify_session_token(x_auth_token), + } diff --git a/backend/agenteval/web/routers/reports.py b/backend/agenteval/web/routers/reports.py index 1c314f4..ea09dac 100644 --- a/backend/agenteval/web/routers/reports.py +++ b/backend/agenteval/web/routers/reports.py @@ -23,10 +23,14 @@ def get_compare_report( session: Session = Depends(get_db), ) -> dict: repo = RunRepository(session) - if not repo.get(run1): + run_a = repo.get(run1) + run_b = repo.get(run2) + if not run_a: raise HTTPException(status_code=404, detail=f"run not found: {run1}") - if not repo.get(run2): + if not run_b: raise HTTPException(status_code=404, detail=f"run not found: {run2}") + if run_a.scenario_id != run_b.scenario_id: + raise HTTPException(status_code=400, detail="对比报告要求两个运行使用相同场景") return generate_compare_report(run1, run2, session) diff --git a/backend/agenteval/web/routers/runs.py b/backend/agenteval/web/routers/runs.py index 5c1181c..89d8ee4 100644 --- a/backend/agenteval/web/routers/runs.py +++ b/backend/agenteval/web/routers/runs.py @@ -8,7 +8,7 @@ from pydantic import BaseModel from sqlmodel import Session from agenteval.evaluation.engine import EvalEngine -from agenteval.models import EvalRun, RunStatus +from agenteval.models import EvalRun, RunStatus, RunTrigger from agenteval.storage.db import get_session, iso_utc from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository from agenteval.utils.llm import extract_reply_text @@ -22,6 +22,7 @@ router = APIRouter() class StartRunRequest(BaseModel): target_id: str scenario_id: str + triggered_by: RunTrigger = RunTrigger.MANUAL # ── Task registry for live evaluation runs ───────────────────────────── @@ -69,7 +70,16 @@ async def _run_evaluation(run_id: str, target_id: str, scenario_id: str) -> None @router.get("") async def list_runs(session: Session = Depends(get_db)) -> list[dict]: - return [r.model_dump() for r in RunRepository(session).list_all()] + scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()} + target_names = {t.id: t.name for t in TargetRepository(session).list_all()} + return [ + { + **r.model_dump(), + "scenario_name": scenario_names.get(r.scenario_id), + "target_name": target_names.get(r.target_id), + } + for r in RunRepository(session).list_all() + ] @router.post("") @@ -82,7 +92,11 @@ async def start_run( if not target or not scenario: raise HTTPException(status_code=404, detail="target or scenario not found") - run = EvalRun(target_id=request.target_id, scenario_id=request.scenario_id) + run = EvalRun( + target_id=request.target_id, + scenario_id=request.scenario_id, + triggered_by=request.triggered_by, + ) run = RunRepository(session).create(run) task = asyncio.create_task( diff --git a/backend/agenteval/web/routers/stats.py b/backend/agenteval/web/routers/stats.py index 23ea630..5876ef3 100644 --- a/backend/agenteval/web/routers/stats.py +++ b/backend/agenteval/web/routers/stats.py @@ -1,34 +1,93 @@ """API routes for statistics and dashboard data.""" from collections import defaultdict +from datetime import datetime, timezone from fastapi import APIRouter, Depends from sqlmodel import Session +from agenteval.storage.model_config_repository import ModelConfigRepository from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository from agenteval.web.deps import get_db router = APIRouter() +def _ts(dt: datetime | None) -> float: + """Sortable timestamp tolerant of naive/aware mixes in legacy rows.""" + if dt is None: + return 0.0 + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.timestamp() + + @router.get("/dashboard") def dashboard(session: Session = Depends(get_db)) -> dict: targets = TargetRepository(session).list_all() scenarios = ScenarioRepository(session).list_all() runs = RunRepository(session).list_all() + model_configs = ModelConfigRepository(session).list_all() + + scenario_names = {s.id: s.name for s in scenarios} + target_names = {t.id: t.name for t in targets} completed_runs = [r for r in runs if r.status == "completed" and r.summary] pass_rates = [r.summary.get("pass_rate", 0) for r in completed_runs if isinstance(r.summary, dict)] overall_pass_rate = sum(pass_rates) / len(pass_rates) if pass_rates else None - recent_runs = sorted(runs, key=lambda r: r.started_at or "", reverse=True)[:10] + today = datetime.now(timezone.utc).date() + today_runs = 0 + running_count = 0 + trigger_breakdown: dict[str, int] = defaultdict(int) + for r in runs: + if r.started_at: + started = r.started_at + if started.tzinfo is None: + started = started.replace(tzinfo=timezone.utc) + if started.date() == today: + today_runs += 1 + if r.status in ("running", "pending"): + running_count += 1 + trigger_breakdown[r.triggered_by.value] += 1 + + # Per-scenario aggregation over completed runs. + by_scenario: dict[str, list] = defaultdict(list) + for r in completed_runs: + by_scenario[r.scenario_id].append(r) + scenario_stats = [] + for sid, sruns in by_scenario.items(): + rates = [r.summary.get("pass_rate", 0) for r in sruns if isinstance(r.summary, dict)] + last_run = max(sruns, key=lambda r: _ts(r.started_at)) + scenario_stats.append({ + "scenario_id": sid, + "scenario_name": scenario_names.get(sid, sid[:8]), + "run_count": len(sruns), + "avg_pass_rate": round(sum(rates) / len(rates), 4) if rates else None, + "last_run_at": last_run.started_at.isoformat() if last_run.started_at else None, + }) + scenario_stats.sort(key=lambda s: s["run_count"], reverse=True) + + recent_runs = sorted(runs, key=lambda r: _ts(r.started_at), reverse=True)[:10] return { "targets_count": len(targets), "scenarios_count": len(scenarios), "runs_count": len(runs), + "model_configs_count": len(model_configs), + "today_runs": today_runs, + "running_count": running_count, "overall_pass_rate": overall_pass_rate, - "recent_runs": [r.model_dump() for r in recent_runs], + "trigger_breakdown": dict(trigger_breakdown), + "scenario_stats": scenario_stats, + "recent_runs": [ + { + **r.model_dump(), + "scenario_name": scenario_names.get(r.scenario_id), + "target_name": target_names.get(r.target_id), + } + for r in recent_runs + ], } diff --git a/backend/cli/run.py b/backend/cli/run.py index 05d3306..be595f9 100644 --- a/backend/cli/run.py +++ b/backend/cli/run.py @@ -5,6 +5,7 @@ from typing import Any import typer from agenteval.evaluation.engine import EvalEngine +from agenteval.models import RunTrigger from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository from rich.console import Console from rich.table import Table @@ -44,7 +45,7 @@ def start_run( raise typer.Exit(1) console.print(f"开始评测: 对象={target.name}, 场景={scenario.name}") - engine = EvalEngine(target=target, scenario=scenario) + engine = EvalEngine(target=target, scenario=scenario, triggered_by=RunTrigger.CLI) try: run = asyncio.run(engine.run(progress_callback=_progress)) except Exception as exc: diff --git a/backend/plugins/openclaw/README.md b/backend/plugins/openclaw/README.md index 7d1aeba..1e0a17c 100644 --- a/backend/plugins/openclaw/README.md +++ b/backend/plugins/openclaw/README.md @@ -1,57 +1,45 @@ # OpenClaw 插件 for AgentEvalTool -本目录提供 OpenClaw 调用 AgentEvalTool 的示例技能，使 OpenClaw 能够自闭环完成评测策略设计。 +本目录提供 OpenClaw 调用 AgentEvalTool 的标准接入方式，使 OpenClaw（AI 助手）能够自闭环完成评测。 ## 设计原则 -- AgentEvalTool 作为独立的 CLI 工具集存在，负责： +- AgentEvalTool 负责： - 评测对象管理 - 评测场景管理 - 评测执行与原始数据收集 - 评估分析与报告生成 -- OpenClaw 通过本插件调用 `agenteval` CLI，负责： +- OpenClaw 负责： - 评测策略编排（选择对象、场景、触发时机） - 定时调度（OpenClaw 自身的 cron + skill 机制） - 结果通知与后续动作 -## 前置条件 +## 唯一标准接入方式：HTTP API -1. 已安装 AgentEvalTool CLI： - ```bash - pip install -e /path/to/AgentEvalTool - ``` -2. `agenteval` 命令已在 PATH 中可用。 -3. 已通过 `agenteval target add` 注册评测对象。 -4. 已通过 `agenteval scenario import` 导入评测场景。 +**所有 OpenClaw 触发的评测必须走 AgentEvalTool 的 HTTP API**（`POST /api/runs` 等）， +并在启动评测时携带 `"triggered_by": "ai_assistant"`，以便平台区分手动触发与 AI 助手触发。 -## 接入方式 +> ⚠️ 禁止在 OpenClaw 中通过 subprocess 调用 `agenteval` CLI。 +> CLI 直接写入其运行环境下的本地 SQLite（路径硬编码），与 Web 后台的数据库不共享， +> 会导致评测记录"消失"——Web 页面永远查不到。CLI 仅用于开发者在 agenteval +> 服务同一环境下的手工调试。 -### 方式一：直接调用 CLI（推荐） +### 方式一：agenteval-run 技能（推荐） -在 OpenClaw Skill 中直接调用系统命令： +`skills/agenteval-run/SKILL.md` 是标准技能文件，指导 AI 助手用 `curl` 走 API 全流程 +（列对象/场景 → 启动评测 → 轮询状态 → 取报告）。 -```python -import subprocess +部署脚本（`scripts/deploy-t480.sh`）会自动把它同步到 OpenClaw 工作区： +`data/openclaw/workspace/skills/agenteval-run/SKILL.md`。 -# 触发评测 -subprocess.run([ - "agenteval", "run", "start", - "--target-id", "", - "--scenario-id", "", -], check=True) +### 方式二：Python Skill 封装 -# 获取报告（需要在运行输出中解析 run_id） -report = subprocess.check_output([ - "agenteval", "report", "show", "", "--format", "json", -]) -``` - -### 方式二：使用本目录封装的 Skill - -参考 `agenteval_skill.py`，在 OpenClaw 中注册技能时传入配置： +参考 `agenteval_skill.py`（httpx 调 API），在 OpenClaw 中注册技能时传入配置： ```json { + "api_base_url": "http://agenteval:8000", + "api_key": "<可选，对应 AGENTEVAL_API_KEY>", "target_id": "", "scenario_id": "", "report_format": "json" @@ -66,20 +54,21 @@ OpenClaw 的 cron 配置（具体格式以 OpenClaw 平台为准）： skill: agenteval_skill schedule: "0 9 * * *" # 每天早上 9 点执行 config: + api_base_url: "http://agenteval:8000" target_id: "" scenario_id: "" report_format: "json" ``` -## CLI 返回的 run_id 解析 +## API 速查 -`agenteval run start` 成功后会输出： +| 操作 | 请求 | +|---|---| +| 启动评测 | `POST /api/runs` body: `{"target_id", "scenario_id", "triggered_by": "ai_assistant"}` | +| 查询状态 | `GET /api/runs/{run_id}`（轮询至 completed/failed） | +| 获取报告 | `GET /api/reports/{run_id}` | -```text -评测完成: run_id=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx, status=completed -``` - -OpenClaw Skill 需要从 stdout 中提取 `run_id`，然后调用 `agenteval report show ` 获取报告。 +如果平台设置了 `AGENTEVAL_API_KEY`，所有请求需带 `X-API-Key` 头。 ## 扩展建议 diff --git a/backend/plugins/openclaw/agenteval_skill.py b/backend/plugins/openclaw/agenteval_skill.py index ba6048b..c0d8c47 100644 --- a/backend/plugins/openclaw/agenteval_skill.py +++ b/backend/plugins/openclaw/agenteval_skill.py @@ -44,6 +44,7 @@ class AgentEvalSkill: json={ "target_id": self.target_id, "scenario_id": self.scenario_id, + "triggered_by": "ai_assistant", }, ) if resp.status_code != 200: diff --git a/backend/plugins/openclaw/skills/agenteval-run/SKILL.md b/backend/plugins/openclaw/skills/agenteval-run/SKILL.md new file mode 100644 index 0000000..7643228 --- /dev/null +++ b/backend/plugins/openclaw/skills/agenteval-run/SKILL.md @@ -0,0 +1,54 @@ +--- +name: agenteval-run +description: 触发 AgentEvalTool 评测任务并获取报告 +--- + +当用户要求执行评测时，使用 exec 工具运行以下命令。 +所有评测必须走 AgentEvalTool 标准 HTTP API（禁止直接调 CLI 或操作数据库）。 + +平台可能启用了 API Key 鉴权。每次执行命令前先读取密钥（文件不存在则为空，不影响未启用鉴权的环境）： + +```bash +KEY=$(cat ~/.openclaw/agenteval-api-key 2>/dev/null) +``` + +以下所有 curl 命令都必须带 `-H "X-API-Key: $KEY"`。 + +## 查看评测对象列表 +```bash +curl -s -H "X-API-Key: $KEY" http://agenteval:8000/api/targets | python3 -m json.tool +``` + +## 查看评测场景列表 +```bash +curl -s -H "X-API-Key: $KEY" http://agenteval:8000/api/scenarios | python3 -m json.tool +``` + +## 启动评测 + +必须携带 `"triggered_by": "ai_assistant"`，平台以此区分手动触发与 AI 助手触发： + +```bash +curl -s -X POST http://agenteval:8000/api/runs \ + -H "X-API-Key: $KEY" \ + -H "Content-Type: application/json" \ + -d '{"target_id": "", "scenario_id": "", "triggered_by": "ai_assistant"}' +``` + +响应中的 `id` 字段即 run_id，请务必回报给用户。 + +## 查看运行状态 + +轮询直到 `status` 变为 `completed` 或 `failed`（建议每 5 秒一次）： + +```bash +curl -s -H "X-API-Key: $KEY" http://agenteval:8000/api/runs/ | python3 -m json.tool +``` + +## 获取报告 +```bash +curl -s -H "X-API-Key: $KEY" http://agenteval:8000/api/reports/ | python3 -m json.tool +``` + +请将 、、 替换为实际值。 +完成后向用户汇报：run_id、状态、通过率（summary.pass_rate）以及失败用例摘要。 diff --git a/migrations/versions/b7d4e6f81c22_add_triggered_by_to_eval_runs.py b/migrations/versions/b7d4e6f81c22_add_triggered_by_to_eval_runs.py new file mode 100644 index 0000000..b4a82f0 --- /dev/null +++ b/migrations/versions/b7d4e6f81c22_add_triggered_by_to_eval_runs.py @@ -0,0 +1,29 @@ +"""add triggered_by to eval_runs + +Revision ID: b7d4e6f81c22 +Revises: a64b2f8c9d10 +Create Date: 2026-07-27 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +import sqlmodel # noqa: F401 +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "b7d4e6f81c22" +down_revision: Union[str, Sequence[str], None] = "a64b2f8c9d10" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table("eval_runs") as batch_op: + batch_op.add_column( + sa.Column("triggered_by", sa.String(), nullable=False, server_default="manual") + ) + + +def downgrade() -> None: + with op.batch_alter_table("eval_runs") as batch_op: + batch_op.drop_column("triggered_by") diff --git a/pyproject.toml b/pyproject.toml index ef20ee4..5e0d592 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agenteval" -version = "0.3.0-dev" +version = "0.4.0-dev" description = "智能体质量评估工具集平台" readme = "README.md" requires-python = ">=3.10" diff --git a/scripts/deploy-t480.sh b/scripts/deploy-t480.sh index 2d6b0fe..0e1a3c7 100755 --- a/scripts/deploy-t480.sh +++ b/scripts/deploy-t480.sh @@ -92,6 +92,23 @@ run rsync -az --delete \ --exclude='*.db*' \ ./ "$HOST:$REMOTE_DIR/" +# ── OpenClaw skill sync ──────────────────────────────────────────────── +# The agenteval-run skill lives in the openclaw workspace volume (data/), +# which rsync excludes. Sync the versioned copy explicitly so the AI +# assistant always follows the standard HTTP API flow. +SKILL_SRC="backend/plugins/openclaw/skills/agenteval-run/SKILL.md" +SKILL_DST="$REMOTE_DIR/data/openclaw/workspace/skills/agenteval-run/SKILL.md" +log "sync OpenClaw skill → $SKILL_DST" +run ssh "$HOST" "mkdir -p $(dirname "$SKILL_DST")" +run rsync -az "$SKILL_SRC" "$HOST:$SKILL_DST" + +# Provision the API key file the skill reads (~/.openclaw/agenteval-api-key in +# the openclaw container = data/openclaw/agenteval-api-key on the host). Sourced +# from AGENTEVAL_API_KEY in the remote .env; removed when the key is unset. +run ssh "$HOST" "KEY=\$(grep -E '^AGENTEVAL_API_KEY=.+' $REMOTE_DIR/.env 2>/dev/null | cut -d= -f2-); \ + if [ -n \"\$KEY\" ]; then printf '%s' \"\$KEY\" > $REMOTE_DIR/data/openclaw/agenteval-api-key; \ + else rm -f $REMOTE_DIR/data/openclaw/agenteval-api-key; fi" + # ── image rebuild ────────────────────────────────────────────────────── if [[ "$SKIP_BUILD" == "1" ]]; then warn "skipping image rebuild (--skip-build)" diff --git a/tests/integration/test_auth_api.py b/tests/integration/test_auth_api.py new file mode 100644 index 0000000..fa5eab9 --- /dev/null +++ b/tests/integration/test_auth_api.py @@ -0,0 +1,89 @@ +"""Integration tests for the login gate: /api/auth/* and require_auth.""" + +import pytest +from fastapi.testclient import TestClient +from sqlmodel import Session, SQLModel, create_engine + +from agenteval.config.settings import Settings +from agenteval.web.app import app +from agenteval.web.deps import get_db + + +@pytest.fixture() +def client_with_db(tmp_path): + from agenteval.storage.db import ( # noqa: F401 + EvalResultDB, EvalRunDB, EvalTargetDB, ScenarioDB, TurnDB, + ) + engine = create_engine( + f"sqlite:///{tmp_path / 'auth_api.db'}", + connect_args={"check_same_thread": False}, + ) + SQLModel.metadata.create_all(engine) + session = Session(engine) + + def override_get_db(): + try: + yield session + finally: + pass + + app.dependency_overrides[get_db] = override_get_db + client = TestClient(app) + yield client + app.dependency_overrides.clear() + session.close() + + +def _patch_settings(monkeypatch, **kwargs): + settings = Settings(_env_file=None, **kwargs) + from agenteval.web import deps as deps_module + from agenteval.web.routers import auth as auth_module + + monkeypatch.setattr(deps_module, "get_settings", lambda: settings) + monkeypatch.setattr(auth_module, "get_settings", lambda: settings) + return settings + + +def test_no_credentials_configured_is_open(client_with_db, monkeypatch): + _patch_settings(monkeypatch) + assert client_with_db.get("/api/targets").status_code == 200 + status = client_with_db.get("/api/auth/status").json() + assert status == {"auth_required": False, "authenticated": True} + + +def test_login_disabled_returns_400(client_with_db, monkeypatch): + _patch_settings(monkeypatch) + resp = client_with_db.post("/api/auth/login", json={"password": "whatever"}) + assert resp.status_code == 400 + + +def test_admin_password_gates_api(client_with_db, monkeypatch): + _patch_settings(monkeypatch, admin_password="s3cret") + + # No credentials → 401. + assert client_with_db.get("/api/targets").status_code == 401 + + # Wrong password → 401. + assert client_with_db.post("/api/auth/login", json={"password": "nope"}).status_code == 401 + + # Correct password → token that unlocks the API. + token = client_with_db.post("/api/auth/login", json={"password": "s3cret"}).json()["token"] + resp = client_with_db.get("/api/targets", headers={"X-Auth-Token": token}) + assert resp.status_code == 200 + + status = client_with_db.get("/api/auth/status", headers={"X-Auth-Token": token}).json() + assert status == {"auth_required": True, "authenticated": True} + assert client_with_db.get("/api/auth/status").json()["authenticated"] is False + + +def test_api_key_still_works_alongside_login(client_with_db, monkeypatch): + _patch_settings(monkeypatch, admin_password="s3cret", api_key="machine-key") + + assert client_with_db.get("/api/targets").status_code == 401 + assert client_with_db.get("/api/targets", headers={"X-API-Key": "machine-key"}).status_code == 200 + assert client_with_db.get("/api/targets", headers={"X-API-Key": "wrong"}).status_code == 401 + + +def test_health_stays_open(client_with_db, monkeypatch): + _patch_settings(monkeypatch, admin_password="s3cret") + assert client_with_db.get("/api/health").status_code == 200 diff --git a/tests/integration/test_reports_api.py b/tests/integration/test_reports_api.py index dc06a87..d778e21 100644 --- a/tests/integration/test_reports_api.py +++ b/tests/integration/test_reports_api.py @@ -39,7 +39,7 @@ def client_with_db(tmp_path): session.close() -def _seed_run(session: Session, name_suffix: str = "") -> str: +def _seed_run(session: Session, name_suffix: str = "", scenario_id: str | None = None) -> str: target = EvalTarget( name=f"target{name_suffix}", platform=PlatformType.AI_DIGITAL_EMPLOYEE, @@ -49,15 +49,17 @@ def _seed_run(session: Session, name_suffix: str = "") -> str: ) target = TargetRepository(session).create(target) - scenario = Scenario( - name=f"scenario{name_suffix}", - cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])], - ) - scenario = ScenarioRepository(session).create(scenario) + if scenario_id is None: + scenario = Scenario( + name=f"scenario{name_suffix}", + cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])], + ) + scenario = ScenarioRepository(session).create(scenario) + scenario_id = scenario.id run = EvalRun( target_id=target.id, - scenario_id=scenario.id, + scenario_id=scenario_id, status=RunStatus.COMPLETED, ) run = RunRepository(session).create(run) @@ -143,7 +145,8 @@ def test_get_markdown_report_attachment_header(client_with_db): def test_compare_report(client_with_db): client, session = client_with_db run_id_a = _seed_run(session, "A") - run_id_b = _seed_run(session, "B") + sid = RunRepository(session).get(run_id_a).scenario_id + run_id_b = _seed_run(session, "B", scenario_id=sid) resp = client.get(f"/api/reports/compare?run1={run_id_a}&run2={run_id_b}") assert resp.status_code == 200 data = resp.json() @@ -153,6 +156,15 @@ def test_compare_report(client_with_db): assert "cases" in data +def test_compare_report_different_scenarios_400(client_with_db): + client, session = client_with_db + run_id_a = _seed_run(session, "A") + run_id_b = _seed_run(session, "B") # separate scenario + resp = client.get(f"/api/reports/compare?run1={run_id_a}&run2={run_id_b}") + assert resp.status_code == 400 + assert "相同场景" in resp.json()["detail"] + + def test_compare_report_run_not_found(client_with_db): client, session = client_with_db run_id = _seed_run(session) diff --git a/tests/integration/test_runs_api.py b/tests/integration/test_runs_api.py index 3e4dd48..3954af7 100644 --- a/tests/integration/test_runs_api.py +++ b/tests/integration/test_runs_api.py @@ -196,3 +196,45 @@ async def test_start_run_missing_target(client, seeded_db): "target_id": "does-not-exist", "scenario_id": "s-1", }) assert resp.status_code == 404 + + +# ── triggered_by ───────────────────────────────────────────────────────── + +async def test_start_run_default_triggered_by_manual(client, seeded_db, mock_channel): + resp = await client.post("/api/runs", json={ + "target_id": "t-1", "scenario_id": "s-1", + }) + assert resp.status_code == 200 + assert resp.json()["triggered_by"] == "manual" + + +async def test_start_run_ai_assistant_triggered_by(client, seeded_db, mock_channel): + resp = await client.post("/api/runs", json={ + "target_id": "t-1", "scenario_id": "s-1", "triggered_by": "ai_assistant", + }) + assert resp.status_code == 200 + run_id = resp.json()["id"] + assert resp.json()["triggered_by"] == "ai_assistant" + + # Persisted, not just echoed. + got = (await client.get(f"/api/runs/{run_id}")).json() + assert got["triggered_by"] == "ai_assistant" + + +async def test_start_run_invalid_triggered_by_422(client, seeded_db): + resp = await client.post("/api/runs", json={ + "target_id": "t-1", "scenario_id": "s-1", "triggered_by": "robot", + }) + assert resp.status_code == 422 + + +async def test_list_runs_includes_names_and_trigger(client, seeded_db, mock_channel): + await client.post("/api/runs", json={ + "target_id": "t-1", "scenario_id": "s-1", "triggered_by": "ai_assistant", + }) + listing = (await client.get("/api/runs")).json() + assert len(listing) == 1 + row = listing[0] + assert row["scenario_name"] == "mock-scenario" + assert row["target_name"] == "mock-target" + assert row["triggered_by"] == "ai_assistant" diff --git a/tests/integration/test_stats_api.py b/tests/integration/test_stats_api.py new file mode 100644 index 0000000..5603868 --- /dev/null +++ b/tests/integration/test_stats_api.py @@ -0,0 +1,106 @@ +"""Integration tests for /api/stats/dashboard aggregation.""" + +import pytest +from fastapi.testclient import TestClient +from sqlmodel import Session, SQLModel, create_engine + +from agenteval.models import ( + Case, CaseType, ChannelType, EvalRun, EvalTarget, PlatformType, + RunStatus, RunTrigger, Scenario, TargetStatus, +) +from agenteval.storage.repository import RunRepository, ScenarioRepository, TargetRepository +from agenteval.web.app import app +from agenteval.web.deps import get_db + + +@pytest.fixture() +def client_with_db(tmp_path): + from agenteval.storage.db import ( # noqa: F401 + EvalResultDB, EvalRunDB, EvalTargetDB, ModelConfigDB, ScenarioDB, TurnDB, + ) + engine = create_engine( + f"sqlite:///{tmp_path / 'stats_api.db'}", + connect_args={"check_same_thread": False}, + ) + SQLModel.metadata.create_all(engine) + session = Session(engine) + + def override_get_db(): + try: + yield session + finally: + pass + + app.dependency_overrides[get_db] = override_get_db + client = TestClient(app) + yield client, session + app.dependency_overrides.clear() + session.close() + + +def _seed(session: Session) -> None: + target = TargetRepository(session).create(EvalTarget( + name="对象A", platform=PlatformType.AI_DIGITAL_EMPLOYEE, + channel_type=ChannelType.TUTU_API, channel_config={}, status=TargetStatus.ACTIVE, + )) + scenario = ScenarioRepository(session).create(Scenario( + name="场景A", cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])], + )) + repo = RunRepository(session) + for pass_rate, trigger in [(1.0, RunTrigger.MANUAL), (0.5, RunTrigger.AI_ASSISTANT)]: + run = repo.create(EvalRun( + target_id=target.id, scenario_id=scenario.id, + status=RunStatus.COMPLETED, triggered_by=trigger, + )) + run.summary = {"total_cases": 2, "passed_cases": 1, "failed_cases": 1, + "total_rules": 2, "passed_rules": 1, "pass_rate": pass_rate} + repo.update(run) + repo.create(EvalRun( + target_id=target.id, scenario_id=scenario.id, + status=RunStatus.RUNNING, triggered_by=RunTrigger.MANUAL, + )) + + +def test_dashboard_aggregates(client_with_db): + client, session = client_with_db + _seed(session) + + data = client.get("/api/stats/dashboard").json() + assert data["targets_count"] == 1 + assert data["scenarios_count"] == 1 + assert data["runs_count"] == 3 + assert data["model_configs_count"] == 0 + assert data["running_count"] == 1 + assert data["today_runs"] == 3 + assert data["overall_pass_rate"] == pytest.approx(0.75) + assert data["trigger_breakdown"] == {"manual": 2, "ai_assistant": 1} + + assert len(data["scenario_stats"]) == 1 + stat = data["scenario_stats"][0] + assert stat["scenario_name"] == "场景A" + assert stat["run_count"] == 2 + assert stat["avg_pass_rate"] == pytest.approx(0.75) + assert stat["last_run_at"] is not None + + assert len(data["recent_runs"]) == 3 + assert data["recent_runs"][0]["scenario_name"] == "场景A" + assert data["recent_runs"][0]["target_name"] == "对象A" + assert "triggered_by" in data["recent_runs"][0] + + +def test_dashboard_empty_db(client_with_db): + client, _ = client_with_db + data = client.get("/api/stats/dashboard").json() + assert data["runs_count"] == 0 + assert data["overall_pass_rate"] is None + assert data["scenario_stats"] == [] + assert data["recent_runs"] == [] + + +def test_trend_returns_daily_points(client_with_db): + client, session = client_with_db + _seed(session) + points = client.get("/api/stats/trend").json() + assert len(points) == 1 + assert points[0]["run_count"] == 2 + assert points[0]["pass_rate"] == pytest.approx(75.0) diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py index dcc93c3..91d9e9b 100644 --- a/tests/unit/test_report.py +++ b/tests/unit/test_report.py @@ -34,7 +34,13 @@ def report_session(tmp_path): engine.dispose() -def _seed_run(session: Session, *, pass_rate: float = 1.0, n_cases: int = 1) -> str: +def _seed_run( + session: Session, + *, + pass_rate: float = 1.0, + n_cases: int = 1, + scenario_id: str | None = None, +) -> str: """Create a minimal completed run with real data in the DB and return run_id.""" target = EvalTarget( name="测试对象", @@ -45,15 +51,17 @@ def _seed_run(session: Session, *, pass_rate: float = 1.0, n_cases: int = 1) -> ) target = TargetRepository(session).create(target) - scenario = Scenario( - name="测试场景", - cases=[Case(id=f"c{i}", type=CaseType.SINGLE, messages=["hi"]) for i in range(n_cases)], - ) - scenario = ScenarioRepository(session).create(scenario) + if scenario_id is None: + scenario = Scenario( + name="测试场景", + cases=[Case(id=f"c{i}", type=CaseType.SINGLE, messages=["hi"]) for i in range(n_cases)], + ) + scenario = ScenarioRepository(session).create(scenario) + scenario_id = scenario.id run = EvalRun( target_id=target.id, - scenario_id=scenario.id, + scenario_id=scenario_id, status=RunStatus.COMPLETED, ) run = RunRepository(session).create(run) @@ -148,9 +156,14 @@ def test_generate_report_not_found_raises(report_session): # ── generate_compare_report ─────────────────────────────────────────────── +def _scenario_of(session: Session, run_id: str) -> str: + return RunRepository(session).get(run_id).scenario_id + + def test_compare_report_structure(report_session): run_id_a = _seed_run(report_session, pass_rate=1.0, n_cases=2) - run_id_b = _seed_run(report_session, pass_rate=0.5, n_cases=2) + sid = _scenario_of(report_session, run_id_a) + run_id_b = _seed_run(report_session, pass_rate=0.5, n_cases=2, scenario_id=sid) result = generate_compare_report(run_id_a, run_id_b, report_session) assert "run_a" in result @@ -163,14 +176,16 @@ def test_compare_report_structure(report_session): def test_compare_report_delta(report_session): run_id_a = _seed_run(report_session, pass_rate=0.5, n_cases=2) - run_id_b = _seed_run(report_session, pass_rate=1.0, n_cases=2) + sid = _scenario_of(report_session, run_id_a) + run_id_b = _seed_run(report_session, pass_rate=1.0, n_cases=2, scenario_id=sid) result = generate_compare_report(run_id_a, run_id_b, report_session) assert result["delta"]["pass_rate"] > 0 # B improved over A def test_compare_report_changed_cases(report_session): run_id_a = _seed_run(report_session, pass_rate=1.0, n_cases=2) - run_id_b = _seed_run(report_session, pass_rate=0.5, n_cases=2) + sid = _scenario_of(report_session, run_id_a) + run_id_b = _seed_run(report_session, pass_rate=0.5, n_cases=2, scenario_id=sid) result = generate_compare_report(run_id_a, run_id_b, report_session) # At least one case changed (A all-pass vs B half-pass) assert result["changed_cases"] >= 1 @@ -178,7 +193,8 @@ def test_compare_report_changed_cases(report_session): def test_compare_report_case_level(report_session): run_id_a = _seed_run(report_session, n_cases=1) - run_id_b = _seed_run(report_session, n_cases=1) + sid = _scenario_of(report_session, run_id_a) + run_id_b = _seed_run(report_session, n_cases=1, scenario_id=sid) result = generate_compare_report(run_id_a, run_id_b, report_session) assert len(result["cases"]) >= 1 case = result["cases"][0] @@ -187,6 +203,13 @@ def test_compare_report_case_level(report_session): assert "changed" in case +def test_compare_report_different_scenarios_rejected(report_session): + run_id_a = _seed_run(report_session, n_cases=1) + run_id_b = _seed_run(report_session, n_cases=1) # separate scenario + with pytest.raises(ValueError, match="same scenario"): + generate_compare_report(run_id_a, run_id_b, report_session) + + # ── render_markdown_report ──────────────────────────────────────────────── def test_render_markdown_contains_header(report_session):