From 13175527015750a3e38eb1bd7e912d8c217d080e Mon Sep 17 00:00:00 2001 From: sinohqb Date: Wed, 5 Aug 2026 03:18:52 +0800 Subject: [PATCH] feat(intelligent-eval): add backend for OpenClaw-driven intelligent evaluation (tickets 01-04) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce 智能评估 as an evaluation paradigm parallel to static evaluation, driven by OpenClaw. The platform supplies storage, lifecycle, and reporting; OpenClaw plans and executes. - Data model: IntelligentEval + Session + Message tables (new, not reusing exploration) - Lifecycle state machine: draft → planning → pending_approval → executing → completed/cancelled/failed - Session API: create/message (channel-forwarded)/close with turn accounting - Report API: pydantic-validated structured report, executing → completed, Markdown export (pure renderer) - Alembic migration for the three tables; domain glossary added to CONTEXT.md --- CONTEXT.md | 30 + .../agenteval/intelligent_eval/__init__.py | 1 + .../agenteval/intelligent_eval/lifecycle.py | 276 ++++++++++ backend/agenteval/intelligent_eval/models.py | 87 +++ backend/agenteval/intelligent_eval/report.py | 130 +++++ .../agenteval/intelligent_eval/repository.py | 231 ++++++++ backend/agenteval/storage/db.py | 94 ++++ backend/agenteval/web/app.py | 2 + .../web/routers/intelligent_evals.py | 244 +++++++++ ...9dbae2a20da_add_intelligent_eval_tables.py | 92 ++++ tests/conftest.py | 3 + tests/integration/test_campaigns_api.py | 3 + tests/integration/test_exploration_api.py | 3 + .../test_exploration_patrol_api.py | 3 + .../integration/test_intelligent_evals_api.py | 511 ++++++++++++++++++ tests/unit/test_intelligent_eval_model.py | 160 ++++++ tests/unit/test_intelligent_eval_report.py | 92 ++++ 17 files changed, 1962 insertions(+) create mode 100644 backend/agenteval/intelligent_eval/__init__.py create mode 100644 backend/agenteval/intelligent_eval/lifecycle.py create mode 100644 backend/agenteval/intelligent_eval/models.py create mode 100644 backend/agenteval/intelligent_eval/report.py create mode 100644 backend/agenteval/intelligent_eval/repository.py create mode 100644 backend/agenteval/web/routers/intelligent_evals.py create mode 100644 migrations/versions/99dbae2a20da_add_intelligent_eval_tables.py create mode 100644 tests/integration/test_intelligent_evals_api.py create mode 100644 tests/unit/test_intelligent_eval_model.py create mode 100644 tests/unit/test_intelligent_eval_report.py diff --git a/CONTEXT.md b/CONTEXT.md index f5ccbc1..97847eb 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -108,6 +108,36 @@ _Avoid_: 评测结果、断言、日志 常驻代理的周期性自主行动：经 OpenClaw heartbeat/cron 唤醒，调平台巡检 API 查看进行中活动的新结果与预算余量，自主决定"继续观察 / 派探索会话"。全局一个常驻巡检作业，无状态地巡检所有活动；OpenClaw 停摆只暂停探索，固定计划照常。 _Avoid_: 轮询、心跳（heartbeat 是 OpenClaw 机制名，巡检是平台侧行为） +## 智能评估 + +**静态评估（Static Evaluation）**: +v0.8 已完整的评测体系：考纲驱动、平台执行、规则判定。预设场景（用例+规则）、平台调度器派生 Run、统计通过率。回答"考纲过了多少"。 +_Avoid_: 固定评估、传统评估 + +**智能评估（Intelligent Evaluation）**: +与静态评估并列的独立评测体系：目标驱动、OpenClaw 规划执行、AI 判定。用户只给方向（目标+种子+意图+角色），OpenClaw 全权规划与执行，产出可驱动被评对象改善的结构化报告。独立实体（IntelligentEval），与 Campaign 平级，不共享数据表与状态机。 +_Avoid_: 动态评估、自动评估 + +**粗计划（Coarse Plan）**: +OpenClaw 在智能评估中产出的评估规划：评估维度、虚拟用户列表、时间分布编排、预算、完成标准。入库可见，用户审批后才执行。"粗"在于只定方向不定细节——具体对话策略、追问节奏、何时放弃由执行时动态决定。 +_Avoid_: 计划、方案（太泛） + +**智能评估会话（IntelligentEvalSession）**: +OpenClaw 以虚拟用户身份与被评对象的一段完整对话，全新实体（不复用 exploration_sessions）。归属智能评估、含人设/目标/维度、会话级评估（verdict）。 +_Avoid_: 探索会话（那是静态评估增强层的概念）、运行 + +**结构化报告（Structured Report）**: +智能评估的最终产出：发现清单（问题+证据+严重程度+改善建议）+ 亮点 + 优先级建议。消费者有两个：人（看问题）和 AI（拿报告去改被评对象的提示词/SOP，驱动其进化）。 +_Avoid_: 评估报告（与静态评估的报告混淆）、分析（与 Analysis 岗位混淆） + +**评估角色（Evaluation Role）**: +OpenClaw 在智能评估中按职责拆分的三个角色：规划师（planner，产出粗计划）、评估者（evaluator，执行对话）、分析师（analyst，汇总报告）。每个角色可配置不同大模型，对应三个独立技能文件。 +_Avoid_: 岗位（与模型用途的"岗位"概念冲突） + +**时间窗口（Time Window）**: +智能评估的模拟约束：模拟一个完整服务周期（如 24h）内的用户交互分布。OpenClaw 规划时考虑交互时机（早高峰、午间冷清、晚间投诉多），通过 cron 自唤醒在对应时间点执行。是模拟约束而非硬截止。 +_Avoid_: 窗口（与静态评估的"服务周期窗口"混淆时需加前缀） + ## 模型配置 **模型能力（Capability）**: diff --git a/backend/agenteval/intelligent_eval/__init__.py b/backend/agenteval/intelligent_eval/__init__.py new file mode 100644 index 0000000..b2f3fd9 --- /dev/null +++ b/backend/agenteval/intelligent_eval/__init__.py @@ -0,0 +1 @@ +"""Intelligent evaluation (智能评估) — OpenClaw-driven independent evaluation.""" diff --git a/backend/agenteval/intelligent_eval/lifecycle.py b/backend/agenteval/intelligent_eval/lifecycle.py new file mode 100644 index 0000000..042b32e --- /dev/null +++ b/backend/agenteval/intelligent_eval/lifecycle.py @@ -0,0 +1,276 @@ +"""Intelligent eval lifecycle (状态机 + 领域操作). + +状态机： + draft → planning → pending_approval → executing → completed + → cancelled + → failed + pending_approval 可打回 → planning（附反馈） + executing 可取消 → cancelled + +非法转换抛 IntelligentEvalTransitionError，路由层映射为 409。 +""" + +import json +from typing import Any + +from sqlmodel import Session + +from agenteval.channels.factory import ChannelFactory +from agenteval.config import get_settings +from agenteval.intelligent_eval.models import ( + IntelligentEval, + IntelligentEvalMessage, + IntelligentEvalSession, + IntelligentEvalSessionStatus, + IntelligentEvalStatus, +) +from agenteval.intelligent_eval.repository import ( + IntelligentEvalMessageRepository, + IntelligentEvalRepository, + IntelligentEvalSessionRepository, +) +from agenteval.storage.db import utc_now +from agenteval.storage.repository import TargetRepository + +# 合法转换表：当前状态 → 允许的目标状态集合 +_TRANSITIONS: dict[IntelligentEvalStatus, set[IntelligentEvalStatus]] = { + IntelligentEvalStatus.DRAFT: {IntelligentEvalStatus.PLANNING}, + IntelligentEvalStatus.PLANNING: {IntelligentEvalStatus.PENDING_APPROVAL}, + IntelligentEvalStatus.PENDING_APPROVAL: {IntelligentEvalStatus.EXECUTING, IntelligentEvalStatus.PLANNING, IntelligentEvalStatus.CANCELLED}, + IntelligentEvalStatus.EXECUTING: {IntelligentEvalStatus.COMPLETED, IntelligentEvalStatus.CANCELLED, IntelligentEvalStatus.FAILED}, + IntelligentEvalStatus.COMPLETED: set(), + IntelligentEvalStatus.CANCELLED: set(), + IntelligentEvalStatus.FAILED: set(), +} + + +class IntelligentEvalNotFoundError(Exception): + pass + + +class IntelligentEvalTransitionError(Exception): + def __init__(self, reason: str): + self.reason = reason + super().__init__(reason) + + +class IntelligentEvalChannelError(Exception): + pass + + +def coerce_reply_text(content: Any) -> str: + """Flatten a reply payload to text (tutu returns msgBody as a parsed object).""" + if isinstance(content, str): + return content + if isinstance(content, dict): + for key in ("content", "text", "message"): + value = content.get(key) + if isinstance(value, str) and value: + return value + if content is None: + return "" + return json.dumps(content, ensure_ascii=False) + + +def _get_or_raise(repo: IntelligentEvalRepository, eval_id: str) -> IntelligentEval: + ev = repo.get(eval_id) + if ev is None: + raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found") + return ev + + +def _transition(repo: IntelligentEvalRepository, ev: IntelligentEval, target: IntelligentEvalStatus) -> IntelligentEval: + allowed = _TRANSITIONS.get(ev.status, set()) + if target not in allowed: + raise IntelligentEvalTransitionError( + f"cannot transition from {ev.status.value} to {target.value}" + ) + return repo.transition_status(ev.id, target) + + +def create_eval( + session: Session, + *, + name: str, + target_id: str, + goal: str, + seeds: dict[str, Any], + intent: str, + role_description: str, + time_window_hours: int = 24, +) -> IntelligentEval: + """创建智能评估并直接进入 planning 状态（draft → planning 一步完成）。""" + repo = IntelligentEvalRepository(session) + ev = repo.create(IntelligentEval( + name=name, + target_id=target_id, + status=IntelligentEvalStatus.DRAFT, + goal=goal, + seeds=seeds, + intent=intent, + role_description=role_description, + time_window_hours=time_window_hours, + )) + return _transition(repo, ev, IntelligentEvalStatus.PLANNING) + + +def submit_plan(session: Session, eval_id: str, plan: dict[str, Any]) -> IntelligentEval: + """OpenClaw 提交粗计划：planning → pending_approval。""" + repo = IntelligentEvalRepository(session) + ev = _get_or_raise(repo, eval_id) + ev.plan = plan + ev.plan_feedback = None + ev = repo.update(ev) + return _transition(repo, ev, IntelligentEvalStatus.PENDING_APPROVAL) + + +def approve(session: Session, eval_id: str) -> IntelligentEval: + """用户批准：pending_approval → executing。""" + repo = IntelligentEvalRepository(session) + ev = _get_or_raise(repo, eval_id) + return _transition(repo, ev, IntelligentEvalStatus.EXECUTING) + + +def reject(session: Session, eval_id: str, feedback: str) -> IntelligentEval: + """用户打回：pending_approval → planning（附反馈）。""" + repo = IntelligentEvalRepository(session) + ev = _get_or_raise(repo, eval_id) + ev.plan_feedback = feedback + ev = repo.update(ev) + return _transition(repo, ev, IntelligentEvalStatus.PLANNING) + + +def cancel(session: Session, eval_id: str) -> IntelligentEval: + """用户取消：pending_approval / executing → cancelled。""" + repo = IntelligentEvalRepository(session) + ev = _get_or_raise(repo, eval_id) + return _transition(repo, ev, IntelligentEvalStatus.CANCELLED) + + +def submit_report(session: Session, eval_id: str, report: dict[str, Any]) -> IntelligentEval: + """OpenClaw 提交结构化报告：executing → completed。 + + 报告结构校验在路由层（pydantic），此处只负责落库与状态迁移。 + """ + repo = IntelligentEvalRepository(session) + ev = _get_or_raise(repo, eval_id) + ev.report = report + ev = repo.update(ev) + return _transition(repo, ev, IntelligentEvalStatus.COMPLETED) + + +def get_eval(session: Session, eval_id: str) -> IntelligentEval: + repo = IntelligentEvalRepository(session) + return _get_or_raise(repo, eval_id) + + +def list_evals(session: Session) -> list[IntelligentEval]: + return IntelligentEvalRepository(session).list_all() + + +def _get_session_or_raise(repo: IntelligentEvalSessionRepository, session_id: str) -> IntelligentEvalSession: + obj = repo.get(session_id) + if obj is None: + raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found") + return obj + + +def open_session( + session: Session, + *, + eval_id: str, + persona: dict[str, Any], + goal: str, + dimension: str | None = None, +) -> IntelligentEvalSession: + """创建虚拟用户会话：仅 executing 状态的评估可创建。""" + eval_repo = IntelligentEvalRepository(session) + ev = _get_or_raise(eval_repo, eval_id) + if ev.status != IntelligentEvalStatus.EXECUTING: + raise IntelligentEvalTransitionError( + f"评估不在执行中（当前 {ev.status.value}），无法创建会话" + ) + repo = IntelligentEvalSessionRepository(session) + return repo.create(IntelligentEvalSession( + eval_id=ev.id, + target_id=ev.target_id, + persona=persona, + goal=goal, + dimension=dimension, + )) + + +async def conduct_turn(session: Session, *, session_id: str, content: str) -> dict[str, Any]: + """一轮完整问答：状态检查 → 通道往返 → 双条消息落库 → 轮次自增。""" + repo = IntelligentEvalSessionRepository(session) + obj = _get_session_or_raise(repo, session_id) + if obj.status != IntelligentEvalSessionStatus.RUNNING: + raise IntelligentEvalTransitionError("会话不在进行中，拒收消息") + + target = TargetRepository(session).get(obj.target_id) + if not target: + raise IntelligentEvalNotFoundError("session target not found") + + channel = ChannelFactory.create(target) + sent_at = utc_now() + try: + send_result = await channel.send(content) + except Exception as exc: # channel adapters raise transport-specific errors + raise IntelligentEvalChannelError(f"评测对象通道发送失败: {exc}") from exc + if not send_result.ok: + raise IntelligentEvalChannelError(f"评测对象通道发送失败: {send_result.error}") + + message_repo = IntelligentEvalMessageRepository(session) + message_repo.create(IntelligentEvalMessage( + session_id=obj.id, role="user", content=content, created_at=sent_at + )) + repo.increment_turns(obj.id) + + try: + reply = await channel.poll_reply( + send_result.question_msg_id, + timeout=get_settings().poll_reply_timeout, + ) + except Exception as exc: + raise IntelligentEvalChannelError(f"等待评测对象回复失败: {exc}") from exc + if reply is None: + raise IntelligentEvalChannelError("等待评测对象回复超时") + + received_at = utc_now() + latency_ms = int((received_at - sent_at).total_seconds() * 1000) + reply_text = coerce_reply_text(reply.content) + message_repo.create(IntelligentEvalMessage( + session_id=obj.id, + role="assistant", + content=reply_text, + latency_ms=latency_ms, + created_at=received_at, + )) + return {"reply": reply_text, "latency_ms": latency_ms, "turn_count": obj.turn_count + 1} + + +def close_session(session: Session, *, session_id: str, verdict: dict[str, Any]) -> IntelligentEvalSession: + """关闭会话并记录结论（verdict）；仅 running 会话可关闭。""" + repo = IntelligentEvalSessionRepository(session) + obj = _get_session_or_raise(repo, session_id) + if obj.status != IntelligentEvalSessionStatus.RUNNING: + raise IntelligentEvalTransitionError("会话不在进行中，无法关闭") + closed = repo.close(obj.id, verdict) + if closed is None: + raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found") + return closed + + +def get_session_by_id(session: Session, session_id: str) -> IntelligentEvalSession: + return _get_session_or_raise(IntelligentEvalSessionRepository(session), session_id) + + +def list_sessions(session: Session, eval_id: str) -> list[IntelligentEvalSession]: + eval_repo = IntelligentEvalRepository(session) + _get_or_raise(eval_repo, eval_id) + return IntelligentEvalSessionRepository(session).list_by_eval(eval_id) + + +def list_messages(session: Session, session_id: str) -> list[IntelligentEvalMessage]: + _get_session_or_raise(IntelligentEvalSessionRepository(session), session_id) + return IntelligentEvalMessageRepository(session).list_by_session(session_id) diff --git a/backend/agenteval/intelligent_eval/models.py b/backend/agenteval/intelligent_eval/models.py new file mode 100644 index 0000000..0c8a322 --- /dev/null +++ b/backend/agenteval/intelligent_eval/models.py @@ -0,0 +1,87 @@ +"""Domain models for intelligent evaluation (智能评估). + +Intelligent evaluation is an independent evaluation paradigm parallel to +static evaluation (v0.8). OpenClaw drives planning and execution; the platform +provides data storage, lifecycle management, and reporting. +""" + +from datetime import datetime +from enum import Enum +from typing import Any, Optional + +from pydantic import BaseModel, Field + + +class IntelligentEvalStatus(str, Enum): + DRAFT = "draft" + PLANNING = "planning" + PENDING_APPROVAL = "pending_approval" + EXECUTING = "executing" + COMPLETED = "completed" + CANCELLED = "cancelled" + FAILED = "failed" + + +class IntelligentEvalSessionStatus(str, Enum): + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + EXPIRED = "expired" + + +class IntelligentEvalMessage(BaseModel): + """One chat message inside an intelligent eval session.""" + + id: Optional[str] = None + session_id: str + role: str = "user" + content: str = "" + latency_ms: Optional[int] = None + created_at: Optional[datetime] = None + + +class IntelligentEvalSession(BaseModel): + """A virtual-user session belonging to one intelligent evaluation.""" + + id: Optional[str] = None + eval_id: str + target_id: str + persona: dict[str, Any] = Field(default_factory=dict) + goal: str = "" + dimension: Optional[str] = None + status: IntelligentEvalSessionStatus = IntelligentEvalSessionStatus.RUNNING + verdict: Optional[dict[str, Any]] = None + turn_count: int = 0 + created_at: Optional[datetime] = None + closed_at: Optional[datetime] = None + + +class IntelligentEval(BaseModel): + """An intelligent evaluation instance — independent entity, peer to Campaign.""" + + id: Optional[str] = None + name: str + target_id: str + status: IntelligentEvalStatus = IntelligentEvalStatus.DRAFT + + # User input (四件套) + goal: str = "" + seeds: dict[str, Any] = Field(default_factory=dict) + intent: str = "" + role_description: str = "" + + # Coarse plan (OpenClaw produces) + plan: Optional[dict[str, Any]] = None + plan_feedback: Optional[str] = None + + # Time window + time_window_hours: int = 24 + + # Report + report: Optional[dict[str, Any]] = None + + # Timestamps + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None diff --git a/backend/agenteval/intelligent_eval/report.py b/backend/agenteval/intelligent_eval/report.py new file mode 100644 index 0000000..8795030 --- /dev/null +++ b/backend/agenteval/intelligent_eval/report.py @@ -0,0 +1,130 @@ +"""Structured report for intelligent evaluation (票据 04). + +Two concerns live here so they stay co-located: + +* **Validation** — pydantic models describing the report contract OpenClaw + submits. A malformed body yields a 422 before any state change happens. +* **Rendering** — ``render_report_markdown`` is a pure function (dict in, + string out) with no DB or I/O, so it unit-tests trivially. +""" + +from typing import Any, Optional + +from pydantic import BaseModel, Field, model_validator + + +class ReportEvidence(BaseModel): + session_id: str = "" + turn_index: Optional[int] = None + user_said: str = "" + assistant_replied: str = "" + + +class ReportFinding(BaseModel): + issue: str = Field(min_length=1) + severity: str = Field(min_length=1) + dimension: str = Field(min_length=1) + evidence: list[ReportEvidence] = Field(default_factory=list) + suggestion: Optional[str] = None + related_sop: Optional[str] = None + + +class ReportHighlight(BaseModel): + description: str = "" + dimension: Optional[str] = None + + +class ReportModel(BaseModel): + summary: str = Field(min_length=1) + scores: Optional[dict[str, Any]] = None + findings: list[ReportFinding] = Field(default_factory=list) + highlights: list[ReportHighlight] = Field(default_factory=list) + priority_recommendations: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def _require_findings(self) -> "ReportModel": + if not self.findings: + raise ValueError("findings 不能为空") + return self + + +_SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2} + + +def _severity_rank(severity: str) -> int: + return _SEVERITY_ORDER.get((severity or "").lower(), 99) + + +def render_report_markdown(report: dict[str, Any], *, name: str = "", eval_id: str = "") -> str: + """Render a validated report dict to Markdown. Pure function, no I/O.""" + validated = ReportModel.model_validate(report) + + lines: list[str] = [] + title = name or "智能评估报告" + lines.append(f"# {title}") + lines.append("") + if eval_id: + lines.append(f"> 评估 ID：`{eval_id}`") + lines.append("") + + lines.append("## 总体概述") + lines.append("") + lines.append(validated.summary.strip()) + lines.append("") + + if validated.scores: + lines.append("## 维度评分") + lines.append("") + lines.append("| 维度 | 分数 |") + lines.append("| --- | --- |") + for dimension, score in validated.scores.items(): + lines.append(f"| {dimension} | {score} |") + lines.append("") + + lines.append("## 问题发现") + lines.append("") + findings = sorted(validated.findings, key=lambda f: _severity_rank(f.severity)) + for index, finding in enumerate(findings, start=1): + lines.append(f"### {index}. {finding.issue}") + lines.append("") + lines.append(f"- **严重程度**：{finding.severity}") + lines.append(f"- **维度**：{finding.dimension}") + if finding.suggestion: + lines.append(f"- **建议**：{finding.suggestion}") + if finding.related_sop: + lines.append(f"- **关联 SOP**：{finding.related_sop}") + if finding.evidence: + lines.append("") + lines.append("**证据**：") + lines.append("") + for ev in finding.evidence: + header_bits = [] + if ev.session_id: + header_bits.append(f"会话 `{ev.session_id}`") + if ev.turn_index is not None: + header_bits.append(f"第 {ev.turn_index} 轮") + header = "（" + "，".join(header_bits) + "）" if header_bits else "" + lines.append(f"> {header}") + if ev.user_said: + lines.append(f"> **用户**：{ev.user_said}") + if ev.assistant_replied: + lines.append(f"> **对象**：{ev.assistant_replied}") + lines.append("") + lines.append("") + + if validated.highlights: + lines.append("## 亮点") + lines.append("") + for highlight in validated.highlights: + suffix = f"（{highlight.dimension}）" if highlight.dimension else "" + lines.append(f"- {highlight.description}{suffix}") + lines.append("") + + if validated.priority_recommendations: + lines.append("## 优先改进建议") + lines.append("") + for recommendation in validated.priority_recommendations: + lines.append(f"- {recommendation}") + lines.append("") + + return "\n".join(lines).rstrip() + "\n" diff --git a/backend/agenteval/intelligent_eval/repository.py b/backend/agenteval/intelligent_eval/repository.py new file mode 100644 index 0000000..a8329ae --- /dev/null +++ b/backend/agenteval/intelligent_eval/repository.py @@ -0,0 +1,231 @@ +"""Repository for intelligent evaluation entities.""" + +from typing import Optional + +from sqlmodel import Session, select + +from agenteval.intelligent_eval.models import ( + IntelligentEval, + IntelligentEvalMessage, + IntelligentEvalSession, + IntelligentEvalSessionStatus, + IntelligentEvalStatus, +) +from agenteval.storage.db import ( + IntelligentEvalDB, + IntelligentEvalMessageDB, + IntelligentEvalSessionDB, + get_session, + utc_now, +) + + +class IntelligentEvalRepository: + """CRUD + lifecycle operations for intelligent evaluations.""" + + def __init__(self, session: Optional[Session] = None): + self.session = session or get_session() + + def _to_db(self, obj: IntelligentEval) -> IntelligentEvalDB: + db = IntelligentEvalDB( + id=obj.id, + name=obj.name, + target_id=obj.target_id, + status=obj.status.value if isinstance(obj.status, IntelligentEvalStatus) else obj.status, + goal=obj.goal, + intent=obj.intent, + role_description=obj.role_description, + time_window_hours=obj.time_window_hours, + plan_feedback=obj.plan_feedback, + created_at=obj.created_at, + updated_at=obj.updated_at, + started_at=obj.started_at, + completed_at=obj.completed_at, + ) + db.set_seeds(obj.seeds) + if obj.plan is not None: + db.set_plan(obj.plan) + if obj.report is not None: + db.set_report(obj.report) + return db + + def _from_db(self, db: IntelligentEvalDB) -> IntelligentEval: + return IntelligentEval( + id=db.id, + name=db.name, + target_id=db.target_id, + status=IntelligentEvalStatus(db.status), + goal=db.goal, + seeds=db.get_seeds(), + intent=db.intent, + role_description=db.role_description, + plan=db.get_plan(), + plan_feedback=db.plan_feedback, + time_window_hours=db.time_window_hours, + report=db.get_report(), + created_at=db.created_at, + updated_at=db.updated_at, + started_at=db.started_at, + completed_at=db.completed_at, + ) + + def list_all(self) -> list[IntelligentEval]: + statement = select(IntelligentEvalDB).order_by(IntelligentEvalDB.created_at.desc()) + return [self._from_db(r) for r in self.session.exec(statement).all()] + + def get(self, eval_id: str) -> Optional[IntelligentEval]: + db = self.session.get(IntelligentEvalDB, eval_id) + return self._from_db(db) if db else None + + def create(self, obj: IntelligentEval) -> IntelligentEval: + db = self._to_db(obj) + self.session.add(db) + self.session.commit() + self.session.refresh(db) + return self._from_db(db) + + def update(self, obj: IntelligentEval) -> IntelligentEval: + db = self.session.get(IntelligentEvalDB, obj.id) + if not db: + raise ValueError(f"IntelligentEval {obj.id} not found") + updated = self._to_db(obj) + updated.id = db.id + # Preserve fields that _to_db doesn't set from None + self.session.delete(db) + self.session.add(updated) + self.session.commit() + self.session.refresh(updated) + return self._from_db(updated) + + def transition_status(self, eval_id: str, new_status: IntelligentEvalStatus) -> Optional[IntelligentEval]: + db = self.session.get(IntelligentEvalDB, eval_id) + if not db: + return None + db.status = new_status.value + db.updated_at = utc_now() + if new_status == IntelligentEvalStatus.EXECUTING and db.started_at is None: + db.started_at = utc_now() + if new_status in (IntelligentEvalStatus.COMPLETED, IntelligentEvalStatus.CANCELLED, IntelligentEvalStatus.FAILED): + db.completed_at = utc_now() + self.session.add(db) + self.session.commit() + self.session.refresh(db) + return self._from_db(db) + + def delete(self, eval_id: str) -> bool: + db = self.session.get(IntelligentEvalDB, eval_id) + if not db: + return False + self.session.delete(db) + self.session.commit() + return True + + +class IntelligentEvalSessionRepository: + """CRUD for intelligent eval sessions.""" + + def __init__(self, session: Optional[Session] = None): + self.session = session or get_session() + + def _from_db(self, db: IntelligentEvalSessionDB) -> IntelligentEvalSession: + return IntelligentEvalSession( + id=db.id, + eval_id=db.eval_id, + target_id=db.target_id, + persona=db.get_persona(), + goal=db.goal, + dimension=db.dimension, + status=IntelligentEvalSessionStatus(db.status), + verdict=db.get_verdict(), + turn_count=db.turn_count, + created_at=db.created_at, + closed_at=db.closed_at, + ) + + def list_by_eval(self, eval_id: str) -> list[IntelligentEvalSession]: + statement = ( + select(IntelligentEvalSessionDB) + .where(IntelligentEvalSessionDB.eval_id == eval_id) + .order_by(IntelligentEvalSessionDB.created_at.asc()) + ) + return [self._from_db(r) for r in self.session.exec(statement).all()] + + def get(self, session_id: str) -> Optional[IntelligentEvalSession]: + db = self.session.get(IntelligentEvalSessionDB, session_id) + return self._from_db(db) if db else None + + def create(self, obj: IntelligentEvalSession) -> IntelligentEvalSession: + db = IntelligentEvalSessionDB( + id=obj.id, + eval_id=obj.eval_id, + target_id=obj.target_id, + goal=obj.goal, + dimension=obj.dimension, + status=obj.status.value if isinstance(obj.status, IntelligentEvalSessionStatus) else obj.status, + turn_count=obj.turn_count, + ) + db.set_persona(obj.persona) + if obj.verdict is not None: + db.set_verdict(obj.verdict) + self.session.add(db) + self.session.commit() + self.session.refresh(db) + return self._from_db(db) + + def close(self, session_id: str, verdict: dict, status: IntelligentEvalSessionStatus = IntelligentEvalSessionStatus.COMPLETED) -> Optional[IntelligentEvalSession]: + db = self.session.get(IntelligentEvalSessionDB, session_id) + if not db: + return None + db.status = status.value + db.set_verdict(verdict) + db.closed_at = utc_now() + self.session.add(db) + self.session.commit() + self.session.refresh(db) + return self._from_db(db) + + def increment_turns(self, session_id: str) -> None: + db = self.session.get(IntelligentEvalSessionDB, session_id) + if db: + db.turn_count += 1 + self.session.add(db) + self.session.commit() + + +class IntelligentEvalMessageRepository: + """CRUD for intelligent eval session messages.""" + + def __init__(self, session: Optional[Session] = None): + self.session = session or get_session() + + def _from_db(self, db: IntelligentEvalMessageDB) -> IntelligentEvalMessage: + return IntelligentEvalMessage( + id=db.id, + session_id=db.session_id, + role=db.role, + content=db.content, + latency_ms=db.latency_ms, + created_at=db.created_at, + ) + + def list_by_session(self, session_id: str) -> list[IntelligentEvalMessage]: + statement = ( + select(IntelligentEvalMessageDB) + .where(IntelligentEvalMessageDB.session_id == session_id) + .order_by(IntelligentEvalMessageDB.created_at.asc()) + ) + return [self._from_db(r) for r in self.session.exec(statement).all()] + + def create(self, obj: IntelligentEvalMessage) -> IntelligentEvalMessage: + db = IntelligentEvalMessageDB( + id=obj.id, + session_id=obj.session_id, + role=obj.role, + content=obj.content, + latency_ms=obj.latency_ms, + created_at=obj.created_at, + ) + self.session.add(db) + self.session.commit() + self.session.refresh(db) + return self._from_db(db) diff --git a/backend/agenteval/storage/db.py b/backend/agenteval/storage/db.py index 626d8ad..1b2ad56 100644 --- a/backend/agenteval/storage/db.py +++ b/backend/agenteval/storage/db.py @@ -474,6 +474,100 @@ class FileRecordDB(SQLModel, table=True): category: Optional[FileCategoryDB] = Relationship(back_populates="files") +class IntelligentEvalDB(SQLModel, table=True): + """Intelligent evaluation (智能评估) — independent entity, peer to Campaign.""" + + __tablename__ = "intelligent_evals" + + id: Optional[str] = Field(default_factory=new_uuid, primary_key=True) + name: str + target_id: str = Field(foreign_key="eval_targets.id") + status: str = "draft" + + # User input (四件套) + goal: str = "" + seeds: str = "{}" + intent: str = "" + role_description: str = "" + + # Coarse plan (OpenClaw produces) + plan: Optional[str] = None + plan_feedback: Optional[str] = None + + # Time window + time_window_hours: int = 24 + + # Report + report: Optional[str] = None + + # Timestamps + created_at: Optional[datetime] = Field(default_factory=utc_now) + updated_at: Optional[datetime] = Field(default_factory=utc_now) + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + + def get_seeds(self) -> dict[str, Any]: + return _json_loads(self.seeds) + + def set_seeds(self, seeds: dict[str, Any]) -> None: + self.seeds = _json_dumps(seeds) + + def get_plan(self) -> Optional[dict[str, Any]]: + return _json_loads(self.plan) if self.plan else None + + def set_plan(self, plan: dict[str, Any]) -> None: + self.plan = _json_dumps(plan) + + def get_report(self) -> Optional[dict[str, Any]]: + return _json_loads(self.report) if self.report else None + + def set_report(self, report: dict[str, Any]) -> None: + self.report = _json_dumps(report) + + +class IntelligentEvalSessionDB(SQLModel, table=True): + """A virtual-user session within an intelligent evaluation.""" + + __tablename__ = "intelligent_eval_sessions" + + id: Optional[str] = Field(default_factory=new_uuid, primary_key=True) + eval_id: str = Field(index=True, foreign_key="intelligent_evals.id") + target_id: str = Field(foreign_key="eval_targets.id") + persona: str = "{}" + goal: str = "" + dimension: Optional[str] = None + status: str = "running" + verdict: Optional[str] = None + turn_count: int = 0 + created_at: Optional[datetime] = Field(default_factory=utc_now) + closed_at: Optional[datetime] = None + + def get_persona(self) -> dict[str, Any]: + return _json_loads(self.persona) + + def set_persona(self, persona: dict[str, Any]) -> None: + self.persona = _json_dumps(persona) + + def get_verdict(self) -> Optional[dict[str, Any]]: + return _json_loads(self.verdict) if self.verdict else None + + def set_verdict(self, verdict: dict[str, Any]) -> None: + self.verdict = _json_dumps(verdict) + + +class IntelligentEvalMessageDB(SQLModel, table=True): + """One chat message inside an intelligent eval session.""" + + __tablename__ = "intelligent_eval_messages" + + id: Optional[str] = Field(default_factory=new_uuid, primary_key=True) + session_id: str = Field(index=True, foreign_key="intelligent_eval_sessions.id") + role: str = "user" + content: str = "" + latency_ms: Optional[int] = None + created_at: Optional[datetime] = Field(default_factory=utc_now) + + def init_db() -> None: SQLModel.metadata.create_all(engine) diff --git a/backend/agenteval/web/app.py b/backend/agenteval/web/app.py index f1cc99f..e2e0315 100644 --- a/backend/agenteval/web/app.py +++ b/backend/agenteval/web/app.py @@ -22,6 +22,7 @@ from agenteval.web.routers import ( campaigns, exploration, files, + intelligent_evals, model_configs, proxy, reports, @@ -102,6 +103,7 @@ app.include_router(scenarios.router, prefix="/api/scenarios", tags=["scenarios"] app.include_router(runs.router, prefix="/api/runs", tags=["runs"], dependencies=_api_deps) app.include_router(campaigns.router, prefix="/api/campaigns", tags=["campaigns"], dependencies=_api_deps) app.include_router(exploration.router, prefix="/api/exploration", tags=["exploration"], dependencies=_api_deps) +app.include_router(intelligent_evals.router, prefix="/api/intelligent-evals", tags=["intelligent-evals"], dependencies=_api_deps) app.include_router(reports.router, prefix="/api/reports", tags=["reports"], dependencies=_api_deps) app.include_router(stats.router, prefix="/api/stats", tags=["stats"], dependencies=_api_deps) app.include_router(files.router, prefix="/api/files", tags=["files"], dependencies=_api_deps) diff --git a/backend/agenteval/web/routers/intelligent_evals.py b/backend/agenteval/web/routers/intelligent_evals.py new file mode 100644 index 0000000..a11d981 --- /dev/null +++ b/backend/agenteval/web/routers/intelligent_evals.py @@ -0,0 +1,244 @@ +"""API routes for intelligent evaluation (智能评估). + +领域逻辑（状态机）在 intelligent_eval/lifecycle.py；本层只做 HTTP 翻译： +NotFound→404、TransitionError→409。 +""" + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import PlainTextResponse +from pydantic import BaseModel, Field +from sqlmodel import Session + +from agenteval.intelligent_eval import lifecycle +from agenteval.intelligent_eval.lifecycle import ( + IntelligentEvalChannelError, + IntelligentEvalNotFoundError, + IntelligentEvalTransitionError, +) +from agenteval.intelligent_eval.report import ReportModel, render_report_markdown +from agenteval.intelligent_eval.repository import IntelligentEvalSessionRepository +from agenteval.web.deps import get_db + +router = APIRouter() + + +class CreateEvalRequest(BaseModel): + name: str = Field(min_length=1) + target_id: str = Field(min_length=1) + goal: str = Field(min_length=1) + seeds: dict[str, Any] = Field(default_factory=dict) + intent: str = "" + role_description: str = "" + time_window_hours: int = Field(default=24, ge=1) + + +class SubmitPlanRequest(BaseModel): + plan: dict[str, Any] + + +class RejectRequest(BaseModel): + feedback: str = Field(min_length=1) + + +class CreateSessionRequest(BaseModel): + persona: dict[str, Any] = Field(default_factory=dict) + goal: str = Field(min_length=1) + dimension: str | None = None + + +class SendMessageRequest(BaseModel): + content: str = Field(min_length=1) + + +class CloseSessionRequest(BaseModel): + verdict: dict[str, Any] + + +class SubmitReportRequest(BaseModel): + report: ReportModel + + +def _translate(exc: Exception) -> HTTPException: + if isinstance(exc, IntelligentEvalNotFoundError): + return HTTPException(status_code=404, detail=str(exc)) + if isinstance(exc, IntelligentEvalChannelError): + return HTTPException(status_code=502, detail=str(exc)) + return HTTPException(status_code=409, detail=exc.reason) + + +def _eval_response(ev, session: Session) -> dict: + data = ev.model_dump(mode="json") + sessions = IntelligentEvalSessionRepository(session).list_by_eval(ev.id) + data["session_count"] = len(sessions) + data["completed_sessions"] = sum(1 for s in sessions if s.status.value == "completed") + return data + + +@router.post("") +async def create_eval(request: CreateEvalRequest, session: Session = Depends(get_db)) -> dict: + ev = lifecycle.create_eval( + session, + name=request.name, + target_id=request.target_id, + goal=request.goal, + seeds=request.seeds, + intent=request.intent, + role_description=request.role_description, + time_window_hours=request.time_window_hours, + ) + return _eval_response(ev, session) + + +@router.get("") +async def list_evals(session: Session = Depends(get_db)) -> dict: + evals = lifecycle.list_evals(session) + return {"intelligent_evals": [_eval_response(ev, session) for ev in evals]} + + +@router.get("/{eval_id}") +async def get_eval(eval_id: str, session: Session = Depends(get_db)) -> dict: + try: + ev = lifecycle.get_eval(session, eval_id) + except IntelligentEvalNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + return _eval_response(ev, session) + + +@router.put("/{eval_id}/plan") +async def submit_plan(eval_id: str, request: SubmitPlanRequest, session: Session = Depends(get_db)) -> dict: + try: + ev = lifecycle.submit_plan(session, eval_id, request.plan) + except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc: + raise _translate(exc) from exc + return _eval_response(ev, session) + + +@router.post("/{eval_id}/approve") +async def approve(eval_id: str, session: Session = Depends(get_db)) -> dict: + try: + ev = lifecycle.approve(session, eval_id) + except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc: + raise _translate(exc) from exc + return _eval_response(ev, session) + + +@router.post("/{eval_id}/reject") +async def reject(eval_id: str, request: RejectRequest, session: Session = Depends(get_db)) -> dict: + try: + ev = lifecycle.reject(session, eval_id, request.feedback) + except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc: + raise _translate(exc) from exc + return _eval_response(ev, session) + + +@router.post("/{eval_id}/cancel") +async def cancel(eval_id: str, session: Session = Depends(get_db)) -> dict: + try: + ev = lifecycle.cancel(session, eval_id) + except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc: + raise _translate(exc) from exc + return _eval_response(ev, session) + + +@router.put("/{eval_id}/report") +async def submit_report(eval_id: str, request: SubmitReportRequest, session: Session = Depends(get_db)) -> dict: + try: + ev = lifecycle.submit_report(session, eval_id, request.report.model_dump()) + except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc: + raise _translate(exc) from exc + return _eval_response(ev, session) + + +@router.get("/{eval_id}/report") +async def get_report(eval_id: str, session: Session = Depends(get_db)) -> dict: + try: + ev = lifecycle.get_eval(session, eval_id) + except IntelligentEvalNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + if ev.report is None: + raise HTTPException(status_code=404, detail="report not submitted yet") + return ev.report + + +@router.get("/{eval_id}/report/markdown", response_class=PlainTextResponse) +async def get_report_markdown(eval_id: str, session: Session = Depends(get_db)) -> PlainTextResponse: + try: + ev = lifecycle.get_eval(session, eval_id) + except IntelligentEvalNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + if ev.report is None: + raise HTTPException(status_code=404, detail="report not submitted yet") + markdown = render_report_markdown(ev.report, name=ev.name, eval_id=ev.id) + return PlainTextResponse(markdown, media_type="text/markdown; charset=utf-8") + + +@router.post("/{eval_id}/sessions") +async def create_session( + eval_id: str, request: CreateSessionRequest, session: Session = Depends(get_db) +) -> dict: + try: + obj = lifecycle.open_session( + session, + eval_id=eval_id, + persona=request.persona, + goal=request.goal, + dimension=request.dimension, + ) + except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc: + raise _translate(exc) from exc + return obj.model_dump(mode="json") + + +@router.get("/{eval_id}/sessions") +async def list_sessions(eval_id: str, session: Session = Depends(get_db)) -> dict: + try: + sessions = lifecycle.list_sessions(session, eval_id) + except IntelligentEvalNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + return {"sessions": [s.model_dump(mode="json") for s in sessions]} + + +def _get_owned_session(eval_id: str, session_id: str, session: Session): + obj = lifecycle.get_session_by_id(session, session_id) + if obj.eval_id != eval_id: + raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found") + return obj + + +@router.post("/{eval_id}/sessions/{session_id}/messages") +async def send_message( + eval_id: str, session_id: str, request: SendMessageRequest, session: Session = Depends(get_db) +) -> dict: + try: + _get_owned_session(eval_id, session_id, session) + return await lifecycle.conduct_turn(session, session_id=session_id, content=request.content) + except ( + IntelligentEvalNotFoundError, + IntelligentEvalTransitionError, + IntelligentEvalChannelError, + ) as exc: + raise _translate(exc) from exc + + +@router.post("/{eval_id}/sessions/{session_id}/close") +async def close_session( + eval_id: str, session_id: str, request: CloseSessionRequest, session: Session = Depends(get_db) +) -> dict: + try: + _get_owned_session(eval_id, session_id, session) + obj = lifecycle.close_session(session, session_id=session_id, verdict=request.verdict) + except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc: + raise _translate(exc) from exc + return obj.model_dump(mode="json") + + +@router.get("/{eval_id}/sessions/{session_id}/messages") +async def list_messages(eval_id: str, session_id: str, session: Session = Depends(get_db)) -> dict: + try: + _get_owned_session(eval_id, session_id, session) + messages = lifecycle.list_messages(session, session_id) + except IntelligentEvalNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + return {"messages": [m.model_dump(mode="json") for m in messages]} diff --git a/migrations/versions/99dbae2a20da_add_intelligent_eval_tables.py b/migrations/versions/99dbae2a20da_add_intelligent_eval_tables.py new file mode 100644 index 0000000..47ab56b --- /dev/null +++ b/migrations/versions/99dbae2a20da_add_intelligent_eval_tables.py @@ -0,0 +1,92 @@ +"""add intelligent eval tables + +Revision ID: 99dbae2a20da +Revises: c5d8f0a2e4b7 +Create Date: 2026-08-05 02:52:58.120905 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision: str = '99dbae2a20da' +down_revision: Union[str, Sequence[str], None] = 'c5d8f0a2e4b7' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('intelligent_evals', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('target_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('goal', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('seeds', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('intent', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('role_description', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('plan', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('plan_feedback', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('time_window_hours', sa.Integer(), nullable=False), + sa.Column('report', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('started_at', sa.DateTime(), nullable=True), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['target_id'], ['eval_targets.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('intelligent_eval_sessions', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('eval_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('target_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('persona', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('goal', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('dimension', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('verdict', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('turn_count', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('closed_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['eval_id'], ['intelligent_evals.id'], ), + sa.ForeignKeyConstraint(['target_id'], ['eval_targets.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('intelligent_eval_sessions', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_intelligent_eval_sessions_eval_id'), ['eval_id'], unique=False) + + op.create_table('intelligent_eval_messages', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('session_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('role', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('content', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('latency_ms', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['session_id'], ['intelligent_eval_sessions.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('intelligent_eval_messages', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_intelligent_eval_messages_session_id'), ['session_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('intelligent_eval_messages', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_intelligent_eval_messages_session_id')) + + op.drop_table('intelligent_eval_messages') + with op.batch_alter_table('intelligent_eval_sessions', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_intelligent_eval_sessions_eval_id')) + + op.drop_table('intelligent_eval_sessions') + op.drop_table('intelligent_evals') + # ### end Alembic commands ### diff --git a/tests/conftest.py b/tests/conftest.py index fd9cc31..0dce9cb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,6 +37,9 @@ def db_session(tmp_db_path: Path) -> Session: EvalTargetDB, ExplorationMessageDB, ExplorationSessionDB, + IntelligentEvalDB, + IntelligentEvalMessageDB, + IntelligentEvalSessionDB, ModelConfigDB, ScenarioDB, ScenarioModelBindingDB, diff --git a/tests/integration/test_campaigns_api.py b/tests/integration/test_campaigns_api.py index 22892e0..7e4b736 100644 --- a/tests/integration/test_campaigns_api.py +++ b/tests/integration/test_campaigns_api.py @@ -481,6 +481,9 @@ def test_exploration_config_migration_on_existing_db(tmp_path, monkeypatch): SQLModel.metadata.create_all(create_engine(database_url)) with create_engine(database_url).begin() as connection: + connection.execute(text("DROP TABLE IF EXISTS intelligent_eval_messages")) + connection.execute(text("DROP TABLE IF EXISTS intelligent_eval_sessions")) + connection.execute(text("DROP TABLE IF EXISTS intelligent_evals")) connection.execute(text("DROP TABLE IF EXISTS exploration_sessions")) connection.execute(text("DROP TABLE IF EXISTS exploration_messages")) connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_seeds")) diff --git a/tests/integration/test_exploration_api.py b/tests/integration/test_exploration_api.py index 59b5516..e4ffd01 100644 --- a/tests/integration/test_exploration_api.py +++ b/tests/integration/test_exploration_api.py @@ -470,6 +470,9 @@ def test_exploration_migration_on_existing_db(tmp_path, monkeypatch): with create_engine(database_url).begin() as connection: from sqlalchemy import text + connection.execute(text("DROP TABLE IF EXISTS intelligent_eval_messages")) + connection.execute(text("DROP TABLE IF EXISTS intelligent_eval_sessions")) + connection.execute(text("DROP TABLE IF EXISTS intelligent_evals")) connection.execute(text("DROP TABLE IF EXISTS exploration_sessions")) connection.execute(text("DROP TABLE IF EXISTS exploration_messages")) connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_seeds")) diff --git a/tests/integration/test_exploration_patrol_api.py b/tests/integration/test_exploration_patrol_api.py index 97dff07..f4858b7 100644 --- a/tests/integration/test_exploration_patrol_api.py +++ b/tests/integration/test_exploration_patrol_api.py @@ -189,6 +189,9 @@ async def test_patrol_migration_column_on_existing_db(tmp_path, monkeypatch): SQLModel.metadata.create_all(create_engine(database_url)) with create_engine(database_url).begin() as connection: + connection.execute(text("DROP TABLE IF EXISTS intelligent_eval_messages")) + connection.execute(text("DROP TABLE IF EXISTS intelligent_eval_sessions")) + connection.execute(text("DROP TABLE IF EXISTS intelligent_evals")) connection.execute(text("DROP TABLE IF EXISTS exploration_sessions")) connection.execute(text("DROP TABLE IF EXISTS exploration_messages")) connection.execute(text("ALTER TABLE campaigns DROP COLUMN exploration_seeds")) diff --git a/tests/integration/test_intelligent_evals_api.py b/tests/integration/test_intelligent_evals_api.py new file mode 100644 index 0000000..462035a --- /dev/null +++ b/tests/integration/test_intelligent_evals_api.py @@ -0,0 +1,511 @@ +"""Integration tests for intelligent eval lifecycle + session + report API (tickets 02, 03, 04). + +Uses httpx.AsyncClient with app= to drive the FastAPI app in-process. +Covers the full lifecycle (create → plan → approve → execute → cancel), +reject/resubmit flow, illegal transition rejections (409), the session +lifecycle (create → message → close) with channel I/O stubbed via MockChannel, +and the report flow (submit → completed → read → Markdown export). +""" + +import pytest +from agenteval.models import ChannelType, EvalTarget, PlatformType, TargetStatus +from agenteval.storage.repository import TargetRepository +from agenteval.web.app import app +from httpx import ASGITransport, AsyncClient + + +@pytest.fixture() +def seeded_db(db_session, monkeypatch): + """Patch get_session/get_db to the test session and seed a target.""" + from agenteval.storage import db as db_module + from agenteval.storage import repository as repo_module + from agenteval.web import app as app_module + + monkeypatch.setattr(app_module, "init_db", lambda: None) + + def _test_get_session(): + return db_session + + monkeypatch.setattr(db_module, "get_session", _test_get_session) + monkeypatch.setattr(repo_module, "get_session", _test_get_session) + + from agenteval.web.deps import get_db + + def _test_get_db(): + try: + yield db_session + finally: + pass + + app.dependency_overrides[get_db] = _test_get_db + + target = EvalTarget( + id="t-1", + name="mock-target", + platform=PlatformType.AI_DIGITAL_EMPLOYEE, + channel_type=ChannelType.TUTU_API, + channel_config={"base_url": "http://mock", "token": "x"}, + status=TargetStatus.ACTIVE, + ) + TargetRepository(db_session).create(target) + + yield db_session + app.dependency_overrides.clear() + + +@pytest.fixture() +async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as c: + yield c + + +async def _create_eval(client, **overrides) -> dict: + payload = { + "name": "客服助手智能评估", + "target_id": "t-1", + "goal": "评估退货流程处理能力", + "seeds": {"personas": ["急躁老客户"], "goals": ["完成退货"]}, + "intent": "考察退货全流程", + "role_description": "模拟真实用户", + "time_window_hours": 24, + } + payload.update(overrides) + resp = await client.post("/api/intelligent-evals", json=payload) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def _submit_plan(client, eval_id: str, plan: dict | None = None) -> dict: + if plan is None: + plan = { + "dimensions": ["退货流程", "投诉处理"], + "virtual_users": [{"persona": {"background": "老客户"}, "goal": "完成退货"}], + "time_distribution": [{"time_slot": "0-2h", "sessions": 1, "scenario": "早间咨询"}], + "estimated_sessions": 3, + "budget": {"max_turns_per_session": 12, "total_max_turns": 36}, + "completion_criteria": "每个维度至少一个会话", + } + resp = await client.put(f"/api/intelligent-evals/{eval_id}/plan", json={"plan": plan}) + assert resp.status_code == 200, resp.text + return resp.json() + + +class TestCreateEval: + async def test_create_transitions_to_planning(self, client, seeded_db): + data = await _create_eval(client) + assert data["status"] == "planning" + assert data["name"] == "客服助手智能评估" + assert data["goal"] == "评估退货流程处理能力" + assert data["time_window_hours"] == 24 + assert "id" in data + + async def test_create_requires_name(self, client, seeded_db): + resp = await client.post("/api/intelligent-evals", json={ + "name": "", "target_id": "t-1", "goal": "test", + }) + assert resp.status_code == 422 + + async def test_create_requires_goal(self, client, seeded_db): + resp = await client.post("/api/intelligent-evals", json={ + "name": "test", "target_id": "t-1", "goal": "", + }) + assert resp.status_code == 422 + + +class TestPlanApproval: + async def test_submit_plan_transitions_to_pending_approval(self, client, seeded_db): + ev = await _create_eval(client) + data = await _submit_plan(client, ev["id"]) + assert data["status"] == "pending_approval" + assert data["plan"]["dimensions"] == ["退货流程", "投诉处理"] + + async def test_approve_transitions_to_executing(self, client, seeded_db): + ev = await _create_eval(client) + await _submit_plan(client, ev["id"]) + resp = await client.post(f"/api/intelligent-evals/{ev['id']}/approve") + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "executing" + assert data["started_at"] is not None + + async def test_reject_transitions_back_to_planning(self, client, seeded_db): + ev = await _create_eval(client) + await _submit_plan(client, ev["id"]) + resp = await client.post(f"/api/intelligent-evals/{ev['id']}/reject", json={"feedback": "缺少投诉维度"}) + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "planning" + assert data["plan_feedback"] == "缺少投诉维度" + + async def test_reject_then_resubmit(self, client, seeded_db): + ev = await _create_eval(client) + await _submit_plan(client, ev["id"], plan={"dimensions": ["A"]}) + await client.post(f"/api/intelligent-evals/{ev['id']}/reject", json={"feedback": "加B"}) + data = await _submit_plan(client, ev["id"], plan={"dimensions": ["A", "B"]}) + assert data["status"] == "pending_approval" + assert data["plan"]["dimensions"] == ["A", "B"] + assert data["plan_feedback"] is None + + +class TestCancel: + async def _create_executing(self, client) -> str: + ev = await _create_eval(client) + await _submit_plan(client, ev["id"]) + await client.post(f"/api/intelligent-evals/{ev['id']}/approve") + return ev["id"] + + async def test_cancel_executing(self, client, seeded_db): + eval_id = await self._create_executing(client) + resp = await client.post(f"/api/intelligent-evals/{eval_id}/cancel") + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "cancelled" + assert data["completed_at"] is not None + + async def test_cancel_pending_approval(self, client, seeded_db): + ev = await _create_eval(client) + await _submit_plan(client, ev["id"]) + resp = await client.post(f"/api/intelligent-evals/{ev['id']}/cancel") + assert resp.status_code == 200 + assert resp.json()["status"] == "cancelled" + + +class TestIllegalTransitions: + async def test_approve_from_planning_returns_409(self, client, seeded_db): + ev = await _create_eval(client) + resp = await client.post(f"/api/intelligent-evals/{ev['id']}/approve") + assert resp.status_code == 409 + + async def test_double_cancel_returns_409(self, client, seeded_db): + ev = await _create_eval(client) + await _submit_plan(client, ev["id"]) + await client.post(f"/api/intelligent-evals/{ev['id']}/approve") + await client.post(f"/api/intelligent-evals/{ev['id']}/cancel") + resp = await client.post(f"/api/intelligent-evals/{ev['id']}/cancel") + assert resp.status_code == 409 + + async def test_submit_plan_from_executing_returns_409(self, client, seeded_db): + ev = await _create_eval(client) + await _submit_plan(client, ev["id"]) + await client.post(f"/api/intelligent-evals/{ev['id']}/approve") + resp = await client.put(f"/api/intelligent-evals/{ev['id']}/plan", json={"plan": {"new": True}}) + assert resp.status_code == 409 + + async def test_not_found_returns_404(self, client, seeded_db): + resp = await client.get("/api/intelligent-evals/nonexistent") + assert resp.status_code == 404 + + +class TestListAndGet: + async def test_list_empty(self, client, seeded_db): + resp = await client.get("/api/intelligent-evals") + assert resp.status_code == 200 + assert resp.json()["intelligent_evals"] == [] + + async def test_list_returns_created(self, client, seeded_db): + await _create_eval(client, name="e1") + await _create_eval(client, name="e2") + resp = await client.get("/api/intelligent-evals") + assert len(resp.json()["intelligent_evals"]) == 2 + + async def test_get_by_id(self, client, seeded_db): + ev = await _create_eval(client, name="e1") + resp = await client.get(f"/api/intelligent-evals/{ev['id']}") + assert resp.status_code == 200 + assert resp.json()["name"] == "e1" + + +async def _create_executing_eval(client) -> str: + ev = await _create_eval(client) + await _submit_plan(client, ev["id"]) + await client.post(f"/api/intelligent-evals/{ev['id']}/approve") + return ev["id"] + + +async def _create_session(client, eval_id: str, **overrides) -> dict: + payload = { + "persona": {"background": "急躁老客户", "style": "直接"}, + "goal": "完成退货", + "dimension": "退货流程", + } + payload.update(overrides) + resp = await client.post(f"/api/intelligent-evals/{eval_id}/sessions", json=payload) + assert resp.status_code == 200, resp.text + return resp.json() + + +class TestCreateSession: + async def test_create_session_in_executing(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + data = await _create_session(client, eval_id) + assert data["eval_id"] == eval_id + assert data["target_id"] == "t-1" + assert data["status"] == "running" + assert data["persona"]["background"] == "急躁老客户" + assert data["dimension"] == "退货流程" + assert data["turn_count"] == 0 + + async def test_create_session_in_planning_returns_409(self, client, seeded_db): + ev = await _create_eval(client) + resp = await client.post(f"/api/intelligent-evals/{ev['id']}/sessions", json={"goal": "g"}) + assert resp.status_code == 409 + + async def test_create_session_requires_goal(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + resp = await client.post(f"/api/intelligent-evals/{eval_id}/sessions", json={"goal": ""}) + assert resp.status_code == 422 + + async def test_create_session_unknown_eval_returns_404(self, client, seeded_db): + resp = await client.post("/api/intelligent-evals/nope/sessions", json={"goal": "g"}) + assert resp.status_code == 404 + + async def test_eval_response_counts_sessions(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + await _create_session(client, eval_id) + resp = await client.get(f"/api/intelligent-evals/{eval_id}") + data = resp.json() + assert data["session_count"] == 1 + assert data["completed_sessions"] == 0 + + +class TestConductTurn: + @pytest.fixture() + def mock_channel(self, monkeypatch): + from agenteval.intelligent_eval import lifecycle as lifecycle_module + + from tests.unit.mock_channel import MockChannel + + channel = MockChannel(reply_text="好的，已为您发起退货申请") + + class _StubFactory: + @staticmethod + def create(target): + return channel + + monkeypatch.setattr(lifecycle_module, "ChannelFactory", _StubFactory) + return channel + + async def test_turn_round_trip_persists_messages(self, client, seeded_db, mock_channel): + eval_id = await _create_executing_eval(client) + session = await _create_session(client, eval_id) + resp = await client.post( + f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages", + json={"content": "我要退货"}, + ) + assert resp.status_code == 200, resp.text + data = resp.json() + assert data["reply"] == "好的，已为您发起退货申请" + assert data["turn_count"] == 1 + + msgs = await client.get(f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages") + messages = msgs.json()["messages"] + assert len(messages) == 2 + assert messages[0]["role"] == "user" + assert messages[0]["content"] == "我要退货" + assert messages[1]["role"] == "assistant" + assert messages[1]["content"] == "好的，已为您发起退货申请" + assert messages[1]["latency_ms"] is not None + + detail = await client.get(f"/api/intelligent-evals/{eval_id}/sessions") + assert detail.json()["sessions"][0]["turn_count"] == 1 + + async def test_turn_increments_across_rounds(self, client, seeded_db, mock_channel): + eval_id = await _create_executing_eval(client) + session = await _create_session(client, eval_id) + for expected in (1, 2, 3): + resp = await client.post( + f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages", + json={"content": f"第{expected}轮"}, + ) + assert resp.json()["turn_count"] == expected + + async def test_turn_unknown_session_returns_404(self, client, seeded_db, mock_channel): + resp = await client.post("/api/intelligent-evals/nope/sessions/nope/messages", json={"content": "hi"}) + assert resp.status_code == 404 + + async def test_channel_send_failure_returns_502(self, client, seeded_db, monkeypatch): + from agenteval.intelligent_eval import lifecycle as lifecycle_module + + from tests.unit.mock_channel import MockChannel + + channel = MockChannel(send_ok=False) + + class _StubFactory: + @staticmethod + def create(target): + return channel + + monkeypatch.setattr(lifecycle_module, "ChannelFactory", _StubFactory) + eval_id = await _create_executing_eval(client) + session = await _create_session(client, eval_id) + resp = await client.post( + f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages", + json={"content": "我要退货"}, + ) + assert resp.status_code == 502 + + +class TestCloseSession: + async def test_close_records_verdict(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + session = await _create_session(client, eval_id) + verdict = {"passed": True, "score": 0.85, "notes": "退货流程顺畅"} + resp = await client.post( + f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close", + json={"verdict": verdict}, + ) + assert resp.status_code == 200, resp.text + data = resp.json() + assert data["status"] == "completed" + assert data["verdict"] == verdict + assert data["closed_at"] is not None + + eval_resp = await client.get(f"/api/intelligent-evals/{eval_id}") + assert eval_resp.json()["completed_sessions"] == 1 + + async def test_double_close_returns_409(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + session = await _create_session(client, eval_id) + await client.post( + f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close", + json={"verdict": {"passed": True}}, + ) + resp = await client.post( + f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close", + json={"verdict": {"passed": True}}, + ) + assert resp.status_code == 409 + + async def test_message_after_close_returns_409(self, client, seeded_db, monkeypatch): + from agenteval.intelligent_eval import lifecycle as lifecycle_module + + from tests.unit.mock_channel import MockChannel + + class _StubFactory: + @staticmethod + def create(target): + return MockChannel() + + monkeypatch.setattr(lifecycle_module, "ChannelFactory", _StubFactory) + eval_id = await _create_executing_eval(client) + session = await _create_session(client, eval_id) + await client.post( + f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/close", + json={"verdict": {"passed": True}}, + ) + resp = await client.post( + f"/api/intelligent-evals/{eval_id}/sessions/{session['id']}/messages", + json={"content": "还在吗"}, + ) + assert resp.status_code == 409 + + async def test_close_unknown_session_returns_404(self, client, seeded_db): + resp = await client.post( + "/api/intelligent-evals/nope/sessions/nope/close", + json={"verdict": {"passed": True}}, + ) + assert resp.status_code == 404 + + +class TestListSessions: + async def test_list_sessions_unknown_eval_returns_404(self, client, seeded_db): + resp = await client.get("/api/intelligent-evals/nope/sessions") + assert resp.status_code == 404 + + async def test_messages_unknown_session_returns_404(self, client, seeded_db): + resp = await client.get("/api/intelligent-evals/nope/sessions/nope/messages") + assert resp.status_code == 404 + + async def test_session_accessed_via_wrong_eval_returns_404(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + session = await _create_session(client, eval_id) + other = await _create_eval(client, name="other") + resp = await client.get(f"/api/intelligent-evals/{other['id']}/sessions/{session['id']}/messages") + assert resp.status_code == 404 + + +def _report_payload() -> dict: + return { + "summary": "整体表现良好。", + "scores": {"退货流程": 0.7}, + "findings": [ + { + "issue": "未主动确认订单号", + "severity": "high", + "dimension": "退货流程", + "evidence": [ + { + "session_id": "s-1", + "turn_index": 3, + "user_said": "我要退货", + "assistant_replied": "好的", + } + ], + "suggestion": "增加确认步骤", + } + ], + "highlights": [{"description": "上下文连贯", "dimension": "多轮追问"}], + "priority_recommendations": ["先修退货确认"], + } + + +class TestReport: + async def test_submit_report_completes_eval(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + resp = await client.put( + f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()} + ) + assert resp.status_code == 200, resp.text + data = resp.json() + assert data["status"] == "completed" + assert data["completed_at"] is not None + assert data["report"]["summary"] == "整体表现良好。" + + async def test_get_report_round_trip(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()}) + resp = await client.get(f"/api/intelligent-evals/{eval_id}/report") + assert resp.status_code == 200 + report = resp.json() + assert report["summary"] == "整体表现良好。" + assert report["findings"][0]["issue"] == "未主动确认订单号" + + async def test_markdown_export(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()}) + resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown") + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("text/markdown") + assert "# 客服助手智能评估" in resp.text + assert "未主动确认订单号" in resp.text + + async def test_submit_report_requires_executing(self, client, seeded_db): + ev = await _create_eval(client) # planning state + resp = await client.put( + f"/api/intelligent-evals/{ev['id']}/report", json={"report": _report_payload()} + ) + assert resp.status_code == 409 + + async def test_submit_report_rejects_invalid_structure(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + bad = {"summary": "缺 findings"} + resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": bad}) + assert resp.status_code == 422 + + async def test_submit_report_rejects_finding_missing_fields(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + payload = _report_payload() + payload["findings"] = [{"issue": "只有 issue"}] + resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": payload}) + assert resp.status_code == 422 + + async def test_get_report_before_submission_returns_404(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + resp = await client.get(f"/api/intelligent-evals/{eval_id}/report") + assert resp.status_code == 404 + + async def test_markdown_before_submission_returns_404(self, client, seeded_db): + eval_id = await _create_executing_eval(client) + resp = await client.get(f"/api/intelligent-evals/{eval_id}/report/markdown") + assert resp.status_code == 404 diff --git a/tests/unit/test_intelligent_eval_model.py b/tests/unit/test_intelligent_eval_model.py new file mode 100644 index 0000000..5fa9be9 --- /dev/null +++ b/tests/unit/test_intelligent_eval_model.py @@ -0,0 +1,160 @@ +"""Smoke tests for intelligent eval data model (ticket 01).""" + +import pytest +from agenteval.intelligent_eval.models import ( + IntelligentEval, + IntelligentEvalMessage, + IntelligentEvalSession, + IntelligentEvalSessionStatus, + IntelligentEvalStatus, +) +from agenteval.intelligent_eval.repository import ( + IntelligentEvalMessageRepository, + IntelligentEvalRepository, + IntelligentEvalSessionRepository, +) +from agenteval.storage.db import EvalTargetDB +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, SQLModel, create_engine + + +@pytest.fixture +def db_session(): + engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) + SQLModel.metadata.create_all(engine) + session = Session(engine) + target = EvalTargetDB(id="t1", name="test-target") + session.add(target) + session.commit() + yield session + session.close() + + +class TestIntelligentEvalRepository: + def test_create_and_get(self, db_session): + repo = IntelligentEvalRepository(db_session) + ev = repo.create(IntelligentEval( + name="test-eval", + target_id="t1", + goal="evaluate customer service", + seeds={"personas": [], "goals": []}, + intent="test intent", + role_description="impatient customer", + time_window_hours=24, + )) + assert ev.id is not None + assert ev.status == IntelligentEvalStatus.DRAFT + assert ev.name == "test-eval" + assert ev.time_window_hours == 24 + + fetched = repo.get(ev.id) + assert fetched is not None + assert fetched.goal == "evaluate customer service" + assert fetched.seeds == {"personas": [], "goals": []} + + def test_list_all(self, db_session): + repo = IntelligentEvalRepository(db_session) + repo.create(IntelligentEval(name="eval-1", target_id="t1")) + repo.create(IntelligentEval(name="eval-2", target_id="t1")) + all_evals = repo.list_all() + assert len(all_evals) == 2 + + def test_status_transitions(self, db_session): + repo = IntelligentEvalRepository(db_session) + ev = repo.create(IntelligentEval(name="eval", target_id="t1")) + assert ev.status == IntelligentEvalStatus.DRAFT + + ev = repo.transition_status(ev.id, IntelligentEvalStatus.PLANNING) + assert ev.status == IntelligentEvalStatus.PLANNING + + ev = repo.transition_status(ev.id, IntelligentEvalStatus.PENDING_APPROVAL) + assert ev.status == IntelligentEvalStatus.PENDING_APPROVAL + + ev = repo.transition_status(ev.id, IntelligentEvalStatus.EXECUTING) + assert ev.status == IntelligentEvalStatus.EXECUTING + assert ev.started_at is not None + + ev = repo.transition_status(ev.id, IntelligentEvalStatus.COMPLETED) + assert ev.status == IntelligentEvalStatus.COMPLETED + assert ev.completed_at is not None + + def test_plan_and_report_json(self, db_session): + repo = IntelligentEvalRepository(db_session) + ev = repo.create(IntelligentEval(name="eval", target_id="t1")) + + plan = {"dimensions": ["退货"], "virtual_users": [], "estimated_sessions": 3} + ev.plan = plan + ev.status = IntelligentEvalStatus.PENDING_APPROVAL + ev = repo.update(ev) + assert ev.plan == plan + + report = {"summary": "good", "findings": []} + ev.report = report + ev = repo.update(ev) + assert ev.report == report + + +class TestIntelligentEvalSessionRepository: + def test_create_and_list(self, db_session): + eval_repo = IntelligentEvalRepository(db_session) + ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) + + sess_repo = IntelligentEvalSessionRepository(db_session) + sess = sess_repo.create(IntelligentEvalSession( + eval_id=ev.id, + target_id="t1", + persona={"name": "user1", "patience": "low"}, + goal="complete return", + dimension="退货流程", + )) + assert sess.id is not None + assert sess.status == IntelligentEvalSessionStatus.RUNNING + assert sess.persona == {"name": "user1", "patience": "low"} + + sessions = sess_repo.list_by_eval(ev.id) + assert len(sessions) == 1 + + def test_close_session(self, db_session): + eval_repo = IntelligentEvalRepository(db_session) + ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) + + sess_repo = IntelligentEvalSessionRepository(db_session) + sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1")) + + verdict = {"goal_achieved": True, "issues": []} + closed = sess_repo.close(sess.id, verdict) + assert closed.status == IntelligentEvalSessionStatus.COMPLETED + assert closed.verdict == verdict + assert closed.closed_at is not None + + def test_increment_turns(self, db_session): + eval_repo = IntelligentEvalRepository(db_session) + ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) + + sess_repo = IntelligentEvalSessionRepository(db_session) + sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1")) + assert sess.turn_count == 0 + + sess_repo.increment_turns(sess.id) + sess_repo.increment_turns(sess.id) + updated = sess_repo.get(sess.id) + assert updated.turn_count == 2 + + +class TestIntelligentEvalMessageRepository: + def test_create_and_list(self, db_session): + eval_repo = IntelligentEvalRepository(db_session) + ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1")) + + sess_repo = IntelligentEvalSessionRepository(db_session) + sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1")) + + msg_repo = IntelligentEvalMessageRepository(db_session) + msg_repo.create(IntelligentEvalMessage(session_id=sess.id, role="user", content="hello")) + msg_repo.create(IntelligentEvalMessage(session_id=sess.id, role="assistant", content="hi", latency_ms=120)) + + messages = msg_repo.list_by_session(sess.id) + assert len(messages) == 2 + assert messages[0].role == "user" + assert messages[1].role == "assistant" + assert messages[1].latency_ms == 120 diff --git a/tests/unit/test_intelligent_eval_report.py b/tests/unit/test_intelligent_eval_report.py new file mode 100644 index 0000000..c82d886 --- /dev/null +++ b/tests/unit/test_intelligent_eval_report.py @@ -0,0 +1,92 @@ +"""Unit tests for the pure Markdown report renderer (票据 04).""" + +import pytest +from agenteval.intelligent_eval.report import render_report_markdown +from pydantic import ValidationError + + +def _report() -> dict: + return { + "summary": "整体表现良好，退货流程存在确认缺失。", + "scores": {"退货流程": 0.7, "投诉处理": 0.85}, + "findings": [ + { + "issue": "未主动确认订单号", + "severity": "high", + "dimension": "退货流程", + "evidence": [ + { + "session_id": "s-1", + "turn_index": 3, + "user_said": "我要退货", + "assistant_replied": "好的，已为您发起", + } + ], + "suggestion": "增加订单号确认步骤", + "related_sop": "退货处理流程 §3.2", + }, + { + "issue": "投诉共情不足", + "severity": "low", + "dimension": "投诉处理", + }, + ], + "highlights": [{"description": "多轮追问保持上下文连贯", "dimension": "多轮追问"}], + "priority_recommendations": ["先修退货确认", "再优化投诉共情"], + } + + +def test_renders_title_and_summary(): + md = render_report_markdown(_report(), name="客服评估", eval_id="e-1") + assert "# 客服评估" in md + assert "整体表现良好" in md + assert "`e-1`" in md + + +def test_renders_scores_table(): + md = render_report_markdown(_report()) + assert "| 维度 | 分数 |" in md + assert "| 退货流程 | 0.7 |" in md + + +def test_sorts_findings_by_severity(): + md = render_report_markdown(_report()) + high_index = md.index("未主动确认订单号") + low_index = md.index("投诉共情不足") + assert high_index < low_index + + +def test_renders_evidence_block(): + md = render_report_markdown(_report()) + assert "会话 `s-1`" in md + assert "第 3 轮" in md + assert "**用户**：我要退货" in md + assert "**对象**：好的，已为您发起" in md + + +def test_renders_highlights_and_recommendations(): + md = render_report_markdown(_report()) + assert "多轮追问保持上下文连贯（多轮追问）" in md + assert "- 先修退货确认" in md + + +def test_minimal_report(): + md = render_report_markdown({"summary": "只有概述", "findings": [{"issue": "i", "severity": "s", "dimension": "d"}]}) + assert "只有概述" in md + assert "## 维度评分" not in md + assert "## 亮点" not in md + + +def test_rejects_missing_summary(): + with pytest.raises(ValidationError): + render_report_markdown({"findings": [{"issue": "i", "severity": "s", "dimension": "d"}]}) + + +def test_rejects_empty_findings(): + with pytest.raises(ValidationError): + render_report_markdown({"summary": "x", "findings": []}) + + +def test_rejects_finding_missing_required_fields(): + with pytest.raises(ValidationError): + render_report_markdown({"summary": "x", "findings": [{"issue": "i"}]})