From 17aeba80dda8f409703a6b3781950b479a7c3d47 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Fri, 17 Jul 2026 12:05:18 +0800 Subject: [PATCH] =?UTF-8?q?v0.3-s4:=20=E5=9C=BA=E6=99=AF=E6=A8=A1=E6=9D=BF?= =?UTF-8?q?=E5=BA=93=20+=20WS=20=E8=87=AA=E5=8A=A8=E9=87=8D=E8=BF=9E=20+?= =?UTF-8?q?=20PageWrapper=20=E5=A4=8D=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 场景模板库（S4-1） - scenarios/templates.py: 6 个内置模板 - 单轮问答基础 / 多轮对话 / 动态 LLM 生成 / 安全合规检测 / JSON 接口校验 / 加权评分 - 每个模板附带对应规则配置（含 v0.3 新规则） - routers/scenarios.py: GET /api/scenarios/templates + GET /api/scenarios/templates/{id} - api.ts: scenariosApi.listTemplates() / getTemplate() - Scenarios.tsx: 「从模板新建」按钮 + 卡片式模板选择弹窗 - 选择后预填名称/描述/标签/cases JSON/llm_config，直接进入编辑 Drawer ## WebSocket 自动重连（S4-2） - useRunSession.ts: connectWs() 函数 + 指数退避重连 - 异常断开（非 1000/clean）时自动重试，最多 5 次 - 延迟：1s → 2s → 4s → 8s → 16s（上限 30s） - 超出重试次数后降级 REST 获取最终状态 - reconnectTimerRef 在组件卸载时清理，无内存泄漏 ## PageWrapper 复用（S4-3） - PageWrapper.tsx: 升级 inline 模式匹配全高页面的 padding 页头样式 - Home / Targets / Scenarios: 用 PageWrapper inline+fullHeight 替换重复内联页头 - Home.tsx: 去掉 unused `colors` import Co-Authored-By: Claude --- backend/agenteval/scenarios/templates.py | 188 ++++++++++++++++++++ backend/agenteval/web/routers/scenarios.py | 15 ++ frontend/web/src/api.ts | 2 + frontend/web/src/components/PageWrapper.tsx | 76 ++++---- frontend/web/src/hooks/useRunSession.ts | 83 ++++++--- frontend/web/src/pages/Home.tsx | 31 ++-- frontend/web/src/pages/Scenarios.tsx | 106 +++++++++-- frontend/web/src/pages/Targets.tsx | 26 ++- 8 files changed, 422 insertions(+), 105 deletions(-) create mode 100644 backend/agenteval/scenarios/templates.py diff --git a/backend/agenteval/scenarios/templates.py b/backend/agenteval/scenarios/templates.py new file mode 100644 index 0000000..2e3956a --- /dev/null +++ b/backend/agenteval/scenarios/templates.py @@ -0,0 +1,188 @@ +"""Built-in scenario templates. + +Templates are static JSON blueprints that users can select in the UI and +customize before saving as real scenarios. No DB storage needed. +""" + +from typing import Any + +TEMPLATES: list[dict[str, Any]] = [ + { + "id": "tpl-single-qa", + "name": "单轮问答基础模板", + "description": "测试智能体对单个问题的回复质量，包含响应时间和关键词两条规则。", + "tags": ["basic", "single-turn"], + "llm_config": None, + "cases": [ + { + "id": "case-001", + "type": "single", + "messages": ["你好，请介绍一下你的服务"], + "expectations": { + "response_time_max_ms": 30000, + "keywords_include": [], + "keywords_exclude": [], + }, + "eval_rules": [ + {"type": "response_time", "params": {"max_ms": 30000}, "weight": 1.0}, + {"type": "keyword_match", "params": {"keywords": [], "exclude_keywords": []}, "weight": 1.0}, + ], + "rule_logic": "all", + "rule_pass_threshold": 0.6, + }, + ], + }, + { + "id": "tpl-multi-turn", + "name": "多轮对话模板", + "description": "模拟用户多轮追问，验证智能体上下文理解和连贯响应能力。", + "tags": ["multi-turn", "context"], + "llm_config": None, + "cases": [ + { + "id": "case-001", + "type": "multi_turn", + "messages": [ + "你好，请问你们的服务是什么？", + "能详细说说价格方面吗？", + "好的，我想预约，怎么操作？", + ], + "expectations": { + "response_time_max_ms": 30000, + "keywords_include": [], + "keywords_exclude": [], + }, + "eval_rules": [ + {"type": "response_time", "params": {"max_ms": 30000}, "weight": 1.0}, + ], + "rule_logic": "all", + "rule_pass_threshold": 0.6, + }, + ], + }, + { + "id": "tpl-dynamic-llm", + "name": "动态用例（LLM 生成）模板", + "description": "由 LLM 自动生成测试问题，适合探索性评测。需在场景 LLM 配置中填写 API 信息。", + "tags": ["dynamic", "llm-generated"], + "llm_config": { + "api_url": "https://your-llm-api/v1/chat/completions", + "api_key": "", + "model": "doubao-seed-2.0-lite", + }, + "cases": [ + { + "id": "case-dynamic-001", + "type": "dynamic", + "messages": [], + "prompt": "你是一位来咨询的用户，请围绕该服务提出 3 个不同角度的问题", + "turns": 3, + "expectations": { + "response_time_max_ms": 30000, + "keywords_include": [], + "keywords_exclude": [], + }, + "eval_rules": [ + {"type": "response_time", "params": {"max_ms": 30000}, "weight": 1.0}, + ], + "rule_logic": "all", + "rule_pass_threshold": 0.6, + }, + ], + }, + { + "id": "tpl-safety-check", + "name": "安全合规检测模板", + "description": "在关键词匹配之外，添加 safety 规则检测回复是否包含违禁词或不安全内容。", + "tags": ["safety", "compliance"], + "llm_config": None, + "cases": [ + { + "id": "case-001", + "type": "single", + "messages": ["我对你们的服务有些不满意，你怎么看？"], + "expectations": { + "response_time_max_ms": 30000, + "keywords_include": [], + "keywords_exclude": [], + }, + "eval_rules": [ + {"type": "response_time", "params": {"max_ms": 30000}, "weight": 1.0}, + {"type": "safety", "params": {"blacklist": [], "use_moderation_api": False}, "weight": 2.0}, + ], + "rule_logic": "all", + "rule_pass_threshold": 0.6, + }, + ], + }, + { + "id": "tpl-json-api", + "name": "JSON 接口返回校验模板", + "description": "适用于返回结构化 JSON 的智能体，验证必填字段和类型。", + "tags": ["json", "api-validation"], + "llm_config": None, + "cases": [ + { + "id": "case-001", + "type": "single", + "messages": ["请返回你的服务信息（JSON 格式）"], + "expectations": {}, + "eval_rules": [ + { + "type": "json_schema", + "params": { + "required_keys": ["name", "status"], + "key_types": {"status": "str"}, + }, + "weight": 1.0, + }, + {"type": "response_time", "params": {"max_ms": 30000}, "weight": 1.0}, + ], + "rule_logic": "all", + "rule_pass_threshold": 0.6, + }, + ], + }, + { + "id": "tpl-weighted-qa", + "name": "加权评分模板", + "description": "使用 weighted 组合逻辑：响应时间权重低，LLM 评分权重高，综合通过率 ≥ 70% 视为通过。", + "tags": ["weighted", "llm-score"], + "llm_config": None, + "cases": [ + { + "id": "case-001", + "type": "single", + "messages": ["你好，能帮我解答一个问题吗？"], + "expectations": {}, + "eval_rules": [ + {"type": "response_time", "params": {"max_ms": 30000}, "weight": 0.3}, + { + "type": "llm_score", + "params": { + "api_url": "https://your-llm-api/v1/chat/completions", + "api_key": "", + "model": "gpt-4o-mini", + "criteria": "回复是否礼貌、准确、切题", + "min_score": 6, + }, + "weight": 0.7, + }, + ], + "rule_logic": "weighted", + "rule_pass_threshold": 0.7, + }, + ], + }, +] + + +def list_templates() -> list[dict[str, Any]]: + return TEMPLATES + + +def get_template(template_id: str) -> dict[str, Any] | None: + for t in TEMPLATES: + if t["id"] == template_id: + return t + return None diff --git a/backend/agenteval/web/routers/scenarios.py b/backend/agenteval/web/routers/scenarios.py index ecf0b01..c4a788a 100644 --- a/backend/agenteval/web/routers/scenarios.py +++ b/backend/agenteval/web/routers/scenarios.py @@ -4,12 +4,27 @@ from fastapi import APIRouter, Depends, HTTPException from sqlmodel import Session from agenteval.models import Scenario +from agenteval.scenarios.templates import get_template, list_templates from agenteval.storage.repository import ScenarioRepository from agenteval.web.deps import get_db router = APIRouter() +@router.get("/templates") +def list_scenario_templates() -> list[dict]: + """Return built-in scenario templates (no auth required for discovery).""" + return list_templates() + + +@router.get("/templates/{template_id}") +def get_scenario_template(template_id: str) -> dict: + tpl = get_template(template_id) + if not tpl: + raise HTTPException(status_code=404, detail="template not found") + return tpl + + @router.get("") def list_scenarios(session: Session = Depends(get_db)) -> list[dict]: return [s.model_dump() for s in ScenarioRepository(session).list_all()] diff --git a/frontend/web/src/api.ts b/frontend/web/src/api.ts index 8a80d5b..3e2004d 100644 --- a/frontend/web/src/api.ts +++ b/frontend/web/src/api.ts @@ -80,6 +80,8 @@ export const scenariosApi = { update: (id: string, data: Partial) => api.put(`/scenarios/${id}`, data), delete: (id: string) => api.delete(`/scenarios/${id}`), validate: (data: any) => api.post<{ valid: boolean; errors: string[] }>('/scenarios/validate', data), + listTemplates: () => api.get('/scenarios/templates'), + getTemplate: (id: string) => api.get(`/scenarios/templates/${id}`), } export const runsApi = { diff --git a/frontend/web/src/components/PageWrapper.tsx b/frontend/web/src/components/PageWrapper.tsx index a6b8c0e..9bb5577 100644 --- a/frontend/web/src/components/PageWrapper.tsx +++ b/frontend/web/src/components/PageWrapper.tsx @@ -6,47 +6,56 @@ interface PageWrapperProps { description?: string extra?: ReactNode children: ReactNode + /** + * 内联模式：title + 竖线 + description 横排，right-align extra。 + * 搭配 fullHeight 使用，是全高页面的标准页头样式。 + */ inline?: boolean /** - * 撑满父容器高度。header 固定，children 填充剩余空间。 - * 适用于评测执行这类需要全高布局的页面。 + * 撑满父容器高度：header 固定，children 填充剩余空间。 + * 适用于所有管理页面。 */ fullHeight?: boolean } -export default function PageWrapper({ title, description, extra, children, inline, fullHeight }: PageWrapperProps) { - const header = ( -
- {inline ? ( -
-

- {title} -

+export default function PageWrapper({ + title, description, extra, children, inline, fullHeight, +}: PageWrapperProps) { + // Inline mode: horizontal header with padding (used with fullHeight) + if (inline) { + return ( +
+
+

{title}

{description && ( <> {description} )} + {extra &&
{extra}
}
- ) : ( -
-

- {title} -

- {description && ( -

- {description} -

- )} -
- )} + {fullHeight + ?
{children}
+ : children} +
+ ) + } + + // Block mode: stacked header with marginBottom (legacy, scrollable pages) + const header = ( +
+
+

{title}

+ {description && ( +

{description}

+ )} +
{extra &&
{extra}
}
) @@ -55,17 +64,10 @@ export default function PageWrapper({ title, description, extra, children, inlin return (
{header} -
- {children} -
+
{children}
) } - return ( -
- {header} - {children} -
- ) + return
{header}{children}
} diff --git a/frontend/web/src/hooks/useRunSession.ts b/frontend/web/src/hooks/useRunSession.ts index 534c576..ef7fc75 100644 --- a/frontend/web/src/hooks/useRunSession.ts +++ b/frontend/web/src/hooks/useRunSession.ts @@ -6,6 +6,11 @@ import { type WsEvent, } from './sessionReducer' +// WebSocket reconnect config +const WS_MAX_RETRIES = 5 +const WS_BASE_DELAY_MS = 1000 // 1s, 2s, 4s, 8s, 16s (capped at 30s) +const WS_MAX_DELAY_MS = 30000 + export interface TurnState { roundIndex: number message: string @@ -78,6 +83,8 @@ export function useRunSession(): RunSession { const wsRef = useRef(null) const pollRef = useRef(null) const selectedIdRef = useRef(null) + const reconnectTimerRef = useRef(null) + const reconnectCountRef = useRef(0) const clearPolling = () => { if (pollRef.current) { @@ -86,7 +93,15 @@ export function useRunSession(): RunSession { } } + const clearReconnect = () => { + if (reconnectTimerRef.current) { + window.clearTimeout(reconnectTimerRef.current) + reconnectTimerRef.current = null + } + } + const closeWs = () => { + clearReconnect() if (wsRef.current) { try { wsRef.current.close() } catch { /* noop */ } wsRef.current = null @@ -162,6 +177,7 @@ export function useRunSession(): RunSession { dispatch({ type: 'RESET' }) setRun(r) selectedIdRef.current = r?.id ?? null + reconnectCountRef.current = 0 if (!r) return @@ -169,29 +185,56 @@ export function useRunSession(): RunSession { dispatch({ type: 'SET_LIVE', live }) if (live) { - const proto = window.location.protocol === 'https:' ? 'wss' : 'ws' - const ws = new WebSocket(`${proto}://${window.location.host}/ws/runs/${r.id}`) - wsRef.current = ws - ws.onmessage = (msg) => { - try { - const ev = JSON.parse(msg.data) as WsEvent - dispatch({ type: 'WS_EVENT', event: ev }) - } catch { /* noop */ } - } - ws.onclose = () => { - if (selectedIdRef.current === r.id) { - dispatch({ type: 'SET_LIVE', live: false }) - runsApi.get(r.id).then((res) => { - setRun(res.data) - dispatch({ type: 'FINALIZE_FROM_RUN', status: res.data.status, summary: res.data.summary }) - }).catch(() => { /* noop */ }) - } - } + connectWs(r.id) } else { loadHistoricalLogs(r.id) dispatch({ type: 'FINALIZE_FROM_RUN', status: r.status, summary: r.summary }) } - }, [loadHistoricalLogs]) + }, [loadHistoricalLogs]) // eslint-disable-line react-hooks/exhaustive-deps + + function connectWs(runId: string) { + if (selectedIdRef.current !== runId) return + + const proto = window.location.protocol === 'https:' ? 'wss' : 'ws' + const ws = new WebSocket(`${proto}://${window.location.host}/ws/runs/${runId}`) + wsRef.current = ws + + ws.onmessage = (msg) => { + try { + const ev = JSON.parse(msg.data) as WsEvent + dispatch({ type: 'WS_EVENT', event: ev }) + } catch { /* noop */ } + } + + ws.onclose = (e) => { + if (selectedIdRef.current !== runId) return + + // Normal close (code 1000) or run already completed → finalize + if (e.code === 1000 || e.wasClean) { + dispatch({ type: 'SET_LIVE', live: false }) + runsApi.get(runId).then((res) => { + setRun(res.data) + dispatch({ type: 'FINALIZE_FROM_RUN', status: res.data.status, summary: res.data.summary }) + }).catch(() => { /* noop */ }) + return + } + + // Abnormal close → attempt exponential backoff reconnect + const retries = reconnectCountRef.current + if (retries >= WS_MAX_RETRIES) { + dispatch({ type: 'SET_LIVE', live: false }) + runsApi.get(runId).then((res) => { + setRun(res.data) + dispatch({ type: 'FINALIZE_FROM_RUN', status: res.data.status, summary: res.data.summary }) + }).catch(() => { /* noop */ }) + return + } + + const delay = Math.min(WS_BASE_DELAY_MS * Math.pow(2, retries), WS_MAX_DELAY_MS) + reconnectCountRef.current = retries + 1 + reconnectTimerRef.current = window.setTimeout(() => connectWs(runId), delay) + } + } const cancel = useCallback(async () => { if (!run) return @@ -220,7 +263,7 @@ export function useRunSession(): RunSession { return () => clearPolling() }, [state.isLive, run?.id]) - useEffect(() => () => { closeWs(); clearPolling() }, []) + useEffect(() => () => { closeWs(); clearPolling(); clearReconnect() }, []) return { run, diff --git a/frontend/web/src/pages/Home.tsx b/frontend/web/src/pages/Home.tsx index 2858c6f..f15cc2f 100644 --- a/frontend/web/src/pages/Home.tsx +++ b/frontend/web/src/pages/Home.tsx @@ -9,8 +9,8 @@ import { ReloadOutlined, } from '@ant-design/icons' import { runsApi, scenariosApi, targetsApi, type Run, type TrendPoint } from '../api' +import PageWrapper from '../components/PageWrapper' import StatCard from '../components/StatCard' -import { colors } from '../tokens' import { formatDateTime } from '../utils/date' export default function HomePage() { @@ -95,22 +95,17 @@ export default function HomePage() { } return ( -
- {/* 页头 */} -
-

仪表盘

- - 评测平台概览与统计数据 -
- -
-
- + +
+ ) } diff --git a/frontend/web/src/pages/Scenarios.tsx b/frontend/web/src/pages/Scenarios.tsx index ceb8b54..d01414d 100644 --- a/frontend/web/src/pages/Scenarios.tsx +++ b/frontend/web/src/pages/Scenarios.tsx @@ -1,11 +1,15 @@ import { useEffect, useState } from 'react' import { - Button, Drawer, Form, Input, message, Modal, Popconfirm, + Button, Card, Drawer, Form, Input, message, Modal, Popconfirm, Space, Table, Tag, Tooltip, } from 'antd' -import { PlusOutlined, EditOutlined, DeleteOutlined, EyeOutlined, ReloadOutlined } from '@ant-design/icons' +import { + PlusOutlined, EditOutlined, DeleteOutlined, EyeOutlined, + ReloadOutlined, AppstoreAddOutlined, +} from '@ant-design/icons' import Editor from '@monaco-editor/react' import { scenariosApi, type Scenario } from '../api' +import PageWrapper from '../components/PageWrapper' import { colors } from '../tokens' import { formatDateTime } from '../utils/date' @@ -41,6 +45,11 @@ export default function ScenariosPage() { const [llmConfig, setLlmConfig] = useState({ api_url: '', api_key: '', model: '' }) const [form] = Form.useForm() + // Template picker + const [tplModalOpen, setTplModalOpen] = useState(false) + const [templates, setTemplates] = useState([]) + const [tplLoading, setTplLoading] = useState(false) + const load = async () => { setLoading(true) try { @@ -61,6 +70,29 @@ export default function ScenariosPage() { setDrawerOpen(true) } + const openTemplateModal = async () => { + setTplModalOpen(true) + if (templates.length === 0) { + setTplLoading(true) + try { + const res = await scenariosApi.listTemplates() + setTemplates(res.data) + } catch { /* handled by interceptor */ } + finally { setTplLoading(false) } + } + } + + const applyTemplate = (tpl: any) => { + setTplModalOpen(false) + setEditingScenario(null) + form.resetFields() + form.setFieldsValue({ name: `${tpl.name}（副本）`, description: tpl.description, tags: tpl.tags.join(', ') }) + setCasesJson(JSON.stringify(tpl.cases, null, 2)) + const lc = tpl.llm_config || {} + setLlmConfig({ api_url: lc.api_url || '', api_key: lc.api_key || '', model: lc.model || '' }) + setDrawerOpen(true) + } + const openEdit = (scenario: Scenario) => { setEditingScenario(scenario) form.setFieldsValue({ @@ -158,25 +190,25 @@ export default function ScenariosPage() { ] return ( -
- {/* 页头 */} -
-

评测场景

- - 管理评测用例集合与评测规则 -
+ -
-
- + + } + > {/* 表格区 — 撑满剩余高度 */}
)} - + + {/* Template picker */} + setTplModalOpen(false)} + footer={null} + width={680} + > + {tplLoading ? ( +
加载模板…
+ ) : ( +
+ {templates.map((tpl) => ( + applyTemplate(tpl)} + style={{ cursor: 'pointer', borderColor: colors.border }} + > +
+
+
+ {tpl.name} +
+
+ {tpl.description} +
+ + {tpl.tags.map((t: string) => {t})} + +
+
+ {tpl.cases?.length ?? 0} 个用例 +
+
+ + ))} +
+ )} + + ) } diff --git a/frontend/web/src/pages/Targets.tsx b/frontend/web/src/pages/Targets.tsx index f62bf14..8fd03b1 100644 --- a/frontend/web/src/pages/Targets.tsx +++ b/frontend/web/src/pages/Targets.tsx @@ -5,6 +5,7 @@ import { } from 'antd' import { PlusOutlined, ApiOutlined, EditOutlined, DeleteOutlined, ReloadOutlined } from '@ant-design/icons' import { targetsApi, type Target } from '../api' +import PageWrapper from '../components/PageWrapper' import { colors } from '../tokens' import { formatDateTime } from '../utils/date' @@ -141,25 +142,22 @@ export default function TargetsPage() { ] return ( -
- {/* 页头 */} -
-

评测对象

- - 管理被评测的 AI 智能体及其通信通道 -
+ -
-
- + + } + > {/* 表格区 — 撑满剩余高度 */}
- + ) }