17aeba80dd
## 场景模板库（S4-1）
- scenarios/templates.py: 6 个内置模板
- 单轮问答基础 / 多轮对话 / 动态 LLM 生成 / 安全合规检测 / JSON 接口校验 / 加权评分
- 每个模板附带对应规则配置（含 v0.3 新规则）
- routers/scenarios.py: GET /api/scenarios/templates + GET /api/scenarios/templates/{id}
- api.ts: scenariosApi.listTemplates() / getTemplate()
- Scenarios.tsx: 「从模板新建」按钮 + 卡片式模板选择弹窗
- 选择后预填名称/描述/标签/cases JSON/llm_config，直接进入编辑 Drawer
## WebSocket 自动重连（S4-2）
- useRunSession.ts: connectWs() 函数 + 指数退避重连
- 异常断开（非 1000/clean）时自动重试，最多 5 次
- 延迟：1s → 2s → 4s → 8s → 16s（上限 30s）
- 超出重试次数后降级 REST 获取最终状态
- reconnectTimerRef 在组件卸载时清理，无内存泄漏
## PageWrapper 复用（S4-3）
- PageWrapper.tsx: 升级 inline 模式匹配全高页面的 padding 页头样式
- Home / Targets / Scenarios: 用 PageWrapper inline+fullHeight 替换重复内联页头
- Home.tsx: 去掉 unused `colors` import
Co-Authored-By: Claude <noreply@anthropic.com>
189 lines
6.9 KiB
Python
189 lines
6.9 KiB
Python
"""Built-in scenario templates.
|
||
|
||
Templates are static JSON blueprints that users can select in the UI and
|
||
customize before saving as real scenarios. No DB storage needed.
|
||
"""
|
||
|
||
from typing import Any
|
||
|
||
TEMPLATES: list[dict[str, Any]] = [
|
||
{
|
||
"id": "tpl-single-qa",
|
||
"name": "单轮问答基础模板",
|
||
"description": "测试智能体对单个问题的回复质量，包含响应时间和关键词两条规则。",
|
||
"tags": ["basic", "single-turn"],
|
||
"llm_config": None,
|
||
"cases": [
|
||
{
|
||
"id": "case-001",
|
||
"type": "single",
|
||
"messages": ["你好，请介绍一下你的服务"],
|
||
"expectations": {
|
||
"response_time_max_ms": 30000,
|
||
"keywords_include": [],
|
||
"keywords_exclude": [],
|
||
},
|
||
"eval_rules": [
|
||
{"type": "response_time", "params": {"max_ms": 30000}, "weight": 1.0},
|
||
{"type": "keyword_match", "params": {"keywords": [], "exclude_keywords": []}, "weight": 1.0},
|
||
],
|
||
"rule_logic": "all",
|
||
"rule_pass_threshold": 0.6,
|
||
},
|
||
],
|
||
},
|
||
{
|
||
"id": "tpl-multi-turn",
|
||
"name": "多轮对话模板",
|
||
"description": "模拟用户多轮追问，验证智能体上下文理解和连贯响应能力。",
|
||
"tags": ["multi-turn", "context"],
|
||
"llm_config": None,
|
||
"cases": [
|
||
{
|
||
"id": "case-001",
|
||
"type": "multi_turn",
|
||
"messages": [
|
||
"你好，请问你们的服务是什么？",
|
||
"能详细说说价格方面吗？",
|
||
"好的，我想预约，怎么操作？",
|
||
],
|
||
"expectations": {
|
||
"response_time_max_ms": 30000,
|
||
"keywords_include": [],
|
||
"keywords_exclude": [],
|
||
},
|
||
"eval_rules": [
|
||
{"type": "response_time", "params": {"max_ms": 30000}, "weight": 1.0},
|
||
],
|
||
"rule_logic": "all",
|
||
"rule_pass_threshold": 0.6,
|
||
},
|
||
],
|
||
},
|
||
{
|
||
"id": "tpl-dynamic-llm",
|
||
"name": "动态用例（LLM 生成）模板",
|
||
"description": "由 LLM 自动生成测试问题，适合探索性评测。需在场景 LLM 配置中填写 API 信息。",
|
||
"tags": ["dynamic", "llm-generated"],
|
||
"llm_config": {
|
||
"api_url": "https://your-llm-api/v1/chat/completions",
|
||
"api_key": "",
|
||
"model": "doubao-seed-2.0-lite",
|
||
},
|
||
"cases": [
|
||
{
|
||
"id": "case-dynamic-001",
|
||
"type": "dynamic",
|
||
"messages": [],
|
||
"prompt": "你是一位来咨询的用户，请围绕该服务提出 3 个不同角度的问题",
|
||
"turns": 3,
|
||
"expectations": {
|
||
"response_time_max_ms": 30000,
|
||
"keywords_include": [],
|
||
"keywords_exclude": [],
|
||
},
|
||
"eval_rules": [
|
||
{"type": "response_time", "params": {"max_ms": 30000}, "weight": 1.0},
|
||
],
|
||
"rule_logic": "all",
|
||
"rule_pass_threshold": 0.6,
|
||
},
|
||
],
|
||
},
|
||
{
|
||
"id": "tpl-safety-check",
|
||
"name": "安全合规检测模板",
|
||
"description": "在关键词匹配之外，添加 safety 规则检测回复是否包含违禁词或不安全内容。",
|
||
"tags": ["safety", "compliance"],
|
||
"llm_config": None,
|
||
"cases": [
|
||
{
|
||
"id": "case-001",
|
||
"type": "single",
|
||
"messages": ["我对你们的服务有些不满意，你怎么看？"],
|
||
"expectations": {
|
||
"response_time_max_ms": 30000,
|
||
"keywords_include": [],
|
||
"keywords_exclude": [],
|
||
},
|
||
"eval_rules": [
|
||
{"type": "response_time", "params": {"max_ms": 30000}, "weight": 1.0},
|
||
{"type": "safety", "params": {"blacklist": [], "use_moderation_api": False}, "weight": 2.0},
|
||
],
|
||
"rule_logic": "all",
|
||
"rule_pass_threshold": 0.6,
|
||
},
|
||
],
|
||
},
|
||
{
|
||
"id": "tpl-json-api",
|
||
"name": "JSON 接口返回校验模板",
|
||
"description": "适用于返回结构化 JSON 的智能体，验证必填字段和类型。",
|
||
"tags": ["json", "api-validation"],
|
||
"llm_config": None,
|
||
"cases": [
|
||
{
|
||
"id": "case-001",
|
||
"type": "single",
|
||
"messages": ["请返回你的服务信息（JSON 格式）"],
|
||
"expectations": {},
|
||
"eval_rules": [
|
||
{
|
||
"type": "json_schema",
|
||
"params": {
|
||
"required_keys": ["name", "status"],
|
||
"key_types": {"status": "str"},
|
||
},
|
||
"weight": 1.0,
|
||
},
|
||
{"type": "response_time", "params": {"max_ms": 30000}, "weight": 1.0},
|
||
],
|
||
"rule_logic": "all",
|
||
"rule_pass_threshold": 0.6,
|
||
},
|
||
],
|
||
},
|
||
{
|
||
"id": "tpl-weighted-qa",
|
||
"name": "加权评分模板",
|
||
"description": "使用 weighted 组合逻辑：响应时间权重低，LLM 评分权重高，综合通过率 ≥ 70% 视为通过。",
|
||
"tags": ["weighted", "llm-score"],
|
||
"llm_config": None,
|
||
"cases": [
|
||
{
|
||
"id": "case-001",
|
||
"type": "single",
|
||
"messages": ["你好，能帮我解答一个问题吗？"],
|
||
"expectations": {},
|
||
"eval_rules": [
|
||
{"type": "response_time", "params": {"max_ms": 30000}, "weight": 0.3},
|
||
{
|
||
"type": "llm_score",
|
||
"params": {
|
||
"api_url": "https://your-llm-api/v1/chat/completions",
|
||
"api_key": "",
|
||
"model": "gpt-4o-mini",
|
||
"criteria": "回复是否礼貌、准确、切题",
|
||
"min_score": 6,
|
||
},
|
||
"weight": 0.7,
|
||
},
|
||
],
|
||
"rule_logic": "weighted",
|
||
"rule_pass_threshold": 0.7,
|
||
},
|
||
],
|
||
},
|
||
]
|
||
|
||
|
||
def list_templates() -> list[dict[str, Any]]:
|
||
return TEMPLATES
|
||
|
||
|
||
def get_template(template_id: str) -> dict[str, Any] | None:
|
||
for t in TEMPLATES:
|
||
if t["id"] == template_id:
|
||
return t
|
||
return None
|