From 855455c0b41704f011c00cfe78cdce22b0ad419a Mon Sep 17 00:00:00 2001 From: sinohqb Date: Fri, 7 Aug 2026 15:53:36 +0800 Subject: [PATCH] feat(deploy): add production release workflow --- deploy/volcengine-102/Dockerfile | 5 +- deploy/volcengine-102/docker-compose.yml | 3 +- docs/deployment/README.md | 1 + docs/deployment/volcengine-102-v1.0.md | 83 +++++++++ scripts/ci-check.sh | 6 +- scripts/deploy-volcengine-102.sh | 203 +++++++++++++++++++++++ scripts/production-entrypoint.sh | 24 +++ 7 files changed, 322 insertions(+), 3 deletions(-) create mode 100644 docs/deployment/volcengine-102-v1.0.md create mode 100755 scripts/deploy-volcengine-102.sh create mode 100755 scripts/production-entrypoint.sh diff --git a/deploy/volcengine-102/Dockerfile b/deploy/volcengine-102/Dockerfile index d33a398..e826813 100644 --- a/deploy/volcengine-102/Dockerfile +++ b/deploy/volcengine-102/Dockerfile @@ -44,4 +44,7 @@ RUN mkdir -p ./data/scenarios ./data/reports EXPOSE 8000 HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ CMD curl -f http://localhost:8000/api/health || exit 1 -CMD ["sh", "-c", "python -c 'from agenteval.storage.db import init_db; init_db()' && alembic stamp head && agenteval server start --host 0.0.0.0 --port 8000"] +COPY scripts/production-entrypoint.sh ./scripts/production-entrypoint.sh +RUN chmod +x ./scripts/production-entrypoint.sh + +CMD ["./scripts/production-entrypoint.sh"] diff --git a/deploy/volcengine-102/docker-compose.yml b/deploy/volcengine-102/docker-compose.yml index d4a15ef..e9fafe5 100644 --- a/deploy/volcengine-102/docker-compose.yml +++ b/deploy/volcengine-102/docker-compose.yml @@ -11,7 +11,8 @@ services: ports: - "8002:8000" env_file: - - .env + - path: .env + required: false volumes: - agenteval_data:/app/data restart: unless-stopped diff --git a/docs/deployment/README.md b/docs/deployment/README.md index 14dad4f..a5c306b 100644 --- a/docs/deployment/README.md +++ b/docs/deployment/README.md @@ -23,6 +23,7 @@ Docker 部署是推荐的生产环境部署方式，提供以下优势： **部署文档**： - [t480 服务器部署指南](t480-v1.0.md) - 针对 t480 测试服务器的具体部署步骤 +- [volcengine-102 正式线部署指南](volcengine-102-v1.0.md) - 版本化发布、备份、迁移、健康检查与回滚 ### 2.2 裸机部署 diff --git a/docs/deployment/volcengine-102-v1.0.md b/docs/deployment/volcengine-102-v1.0.md new file mode 100644 index 0000000..ceedb09 --- /dev/null +++ b/docs/deployment/volcengine-102-v1.0.md @@ -0,0 +1,83 @@ +# volcengine-102 正式线部署指南 + +**状态**：部署流程已实现；正式主机与密钥仍需由运维环境注入。 +**Compose**：`deploy/volcengine-102/docker-compose.yml` +**入口脚本**：`scripts/deploy-volcengine-102.sh` + +## 1. 部署边界 + +正式线使用两个容器：`agenteval`（端口 `8002`）和 `openclaw-eval`（端口 `28789`）。SQLite 数据保存在 Docker named volume 中；生产 `.env` 不从代码仓库同步，必须预先放在远端： + +```text +/opt/agenteval/deploy/volcengine-102/.env +``` + +环境变量模板见 [`deploy/volcengine-102/.env.example`](../../deploy/volcengine-102/.env.example)。正式域名、API Key、OpenClaw token 和 webhook secret 不得写入 Git。 + +## 2. 首次准备 + +在正式主机完成 Docker/Compose、SSH 和镜像仓库访问配置，然后建立远端目录并准备 `.env`。本地只需要配置 SSH 别名或地址： + +```bash +export AGENTEVAL_PROD_HOST=volcengine-102 +export AGENTEVAL_PROD_DIR=/opt/agenteval +export AGENTEVAL_PROD_BACKUP_DIR=/var/backups/agenteval +``` + +首次正式发布前必须确认： + +- 生产 `.env` 已填写非空 `AGENTEVAL_API_KEY`、`AGENTEVAL_SECRET_KEY` 和 OpenClaw token。 +- 生产域名已加入 `AGENTEVAL_ALLOWED_ORIGINS`。 +- 正式主机可拉取 `registry.solahqb22.cn/sola/openclaw:latest`。 +- Docker named volume 和备份目录具备写权限。 + +## 3. 发布流程 + +发布必须基于已提交的 Git 版本；脚本会拒绝存在 tracked 未提交修改的工作区。推荐使用版本号加 commit 作为不可变镜像标签： + +```bash +scripts/deploy-volcengine-102.sh --dry-run --tag 0.8.0-6248568 +scripts/deploy-volcengine-102.sh --tag 0.8.0-6248568 +``` + +脚本依次执行：SSH 检查 → 版本读取 → 远端 `.env` 检查 → 同步已提交源代码 → 备份数据 volume → 远端构建带版本元数据的镜像 → 启动双容器 → 等待健康接口 → 校验版本/commit → 鉴权 API 冒烟。 + +后端启动入口 [`scripts/production-entrypoint.sh`](../../scripts/production-entrypoint.sh) 的规则是： + +- 全新数据库：创建初始 SQLModel schema，记录当前 Alembic 基线，再执行后续迁移。 +- 旧版无 `alembic_version` 的数据库：保留旧数据，记录基线，再执行后续迁移。 +- 已有 Alembic 版本的数据库：只执行 `alembic upgrade head`。 + +## 4. 回滚 + +回滚只切换应用镜像，不自动降级数据库： + +```bash +scripts/deploy-volcengine-102.sh --rollback 0.8.0-previous +``` + +每次发布和回滚前都会在远端备份 named volume 到 `AGENTEVAL_PROD_BACKUP_DIR`。如果某个版本包含不可逆数据迁移，应先停止服务、恢复匹配的数据备份，再启动旧镜像；不得只回滚代码而忽略数据库版本。 + +## 5. 发布后检查 + +```bash +ssh "$AGENTEVAL_PROD_HOST" \ + 'cd /opt/agenteval && docker compose -f deploy/volcengine-102/docker-compose.yml ps' +``` + +至少验证：`/api/health`、`/api/targets`、`/api/scenarios`、`/api/runs`、`/api/campaigns`、`/api/intelligent-evals`、`/api/model-configs` 和 `/openclaw/`。发现异常时保留部署输出、容器日志和备份文件，再执行回滚判断。 + +## 6. 线路关系 + +```mermaid +flowchart LR + Commit[已提交 Git 版本] --> Dev[scripts/deploy-t480.sh] + Dev --> DevCheck[t480 健康检查与业务冒烟] + DevCheck --> Tag[Gitea Tag / 发布审批] + Tag --> Prod[scripts/deploy-volcengine-102.sh] + Prod --> Backup[备份 named volume] + Backup --> Migrate[Alembic upgrade head] + Migrate --> Smoke[正式线 API / OpenClaw 冒烟] + Smoke --> Live[正式服务] + Smoke --> Rollback[旧镜像 + 数据备份恢复] +``` diff --git a/scripts/ci-check.sh b/scripts/ci-check.sh index 577c28e..787b952 100755 --- a/scripts/ci-check.sh +++ b/scripts/ci-check.sh @@ -23,7 +23,11 @@ else fi echo "==> 3/4 pytest" -python -m pytest -q +# Tests use mocked HTTP boundaries and should not inherit a developer's local +# SOCKS/HTTP proxy. Otherwise httpx may require an optional socksio package +# merely while importing the reverse-proxy router. +env -u ALL_PROXY -u all_proxy -u HTTP_PROXY -u http_proxy -u HTTPS_PROXY -u https_proxy \ + python -m pytest -q if [[ $FAST -eq 1 ]]; then echo "==> 4/4 前端类型检查（--fast 跳过）" diff --git a/scripts/deploy-volcengine-102.sh b/scripts/deploy-volcengine-102.sh new file mode 100755 index 0000000..972380d --- /dev/null +++ b/scripts/deploy-volcengine-102.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# Versioned production deployment for the volcengine-102 environment. +# +# Required: +# AGENTEVAL_PROD_HOST= +# +# Optional: +# AGENTEVAL_PROD_DIR=/opt/agenteval +# AGENTEVAL_PROD_PORT=8002 +# AGENTEVAL_PROD_BACKUP_DIR=/var/backups/agenteval +# +# Usage: +# scripts/deploy-volcengine-102.sh --tag 0.8.0-6248568 +# scripts/deploy-volcengine-102.sh --dry-run --tag 0.8.0-6248568 +# scripts/deploy-volcengine-102.sh --rollback 0.8.0-6248568 + +set -euo pipefail + +HOST="${AGENTEVAL_PROD_HOST:-}" +REMOTE_DIR="${AGENTEVAL_PROD_DIR:-/opt/agenteval}" +APP_PORT="${AGENTEVAL_PROD_PORT:-8002}" +BACKUP_DIR="${AGENTEVAL_PROD_BACKUP_DIR:-/var/backups/agenteval}" +COMPOSE_FILE="deploy/volcengine-102/docker-compose.yml" + +DRY_RUN=0 +IMAGE_TAG="" +ROLLBACK_TAG="" + +usage() { + sed -n '2,18p' "$0" +} + +log() { printf '\033[1;36m>> %s\033[0m\n' "$*"; } +warn() { printf '\033[1;33m!! %s\033[0m\n' "$*" >&2; } +die() { printf '\033[1;31m!! %s\033[0m\n' "$*" >&2; exit 1; } +run() { + if [[ "$DRY_RUN" == "1" ]]; then + printf '\033[2m[dry-run]' + printf ' %q' "$@" + printf '\033[0m\n' + else + "$@" + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --tag) + [[ $# -ge 2 ]] || die "--tag requires a value" + IMAGE_TAG="$2" + shift 2 + ;; + --rollback) + [[ $# -ge 2 ]] || die "--rollback requires an image tag" + ROLLBACK_TAG="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$HOST" ]] || die "AGENTEVAL_PROD_HOST is required" +[[ -z "$IMAGE_TAG" || -z "$ROLLBACK_TAG" ]] || die "--tag and --rollback cannot be used together" + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +validate_tag() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || die "invalid image tag: $1" +} + +backup_remote_data() { + local release_tag="$1" + local timestamp backup_file + timestamp=$(date -u +"%Y%m%dT%H%M%SZ") + backup_file="agenteval-${release_tag}-${timestamp}.tgz" + + log "backup production data volume → $BACKUP_DIR/$backup_file" + run ssh "$HOST" "set -eu; \ + mkdir -p '$BACKUP_DIR'; \ + if docker inspect agenteval >/dev/null 2>&1; then \ + DATA_VOLUME=\$(docker inspect agenteval --format '{{range .Mounts}}{{if eq .Destination \"/app/data\"}}{{.Name}}{{end}}{{end}}'); \ + [ -n \"\$DATA_VOLUME\" ] || { echo 'agenteval data volume not found' >&2; exit 1; }; \ + docker run --rm -v \"\$DATA_VOLUME:/source:ro\" -v '$BACKUP_DIR:/backup' alpine:3.20 \ + tar -czf '/backup/$backup_file' -C /source .; \ + else \ + echo 'first deployment: no existing agenteval container, backup skipped'; \ + fi" +} + +wait_for_health() { + local body="" + log "waiting for production /api/health" >&2 + for _attempt in $(seq 1 30); do + body=$(ssh "$HOST" "curl -sf http://localhost:$APP_PORT/api/health 2>/dev/null" || true) + [[ -n "$body" ]] && break + sleep 2 + done + [[ -n "$body" ]] || die "production health check did not respond after 60s" + printf '%s' "$body" +} + +log "pre-flight: ssh connectivity" +run ssh -o ConnectTimeout=10 "$HOST" "echo ok >/dev/null" || die "cannot reach $HOST via ssh" + +if [[ -n "$ROLLBACK_TAG" ]]; then + validate_tag "$ROLLBACK_TAG" + backup_remote_data "pre-rollback-$ROLLBACK_TAG" + log "rollback production image → agenteval:$ROLLBACK_TAG" + run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \ + docker image inspect 'agenteval:$ROLLBACK_TAG' >/dev/null; \ + IMAGE_TAG='$ROLLBACK_TAG' docker compose -f '$COMPOSE_FILE' up -d --no-build agenteval" + if [[ "$DRY_RUN" == "1" ]]; then + log "rollback dry-run complete" + exit 0 + fi + HEALTH_BODY=$(wait_for_health) + log "/api/health: $HEALTH_BODY" + warn "database migrations are not downgraded; restore a matching backup only when the release requires it" + log "rollback OK: image=agenteval:$ROLLBACK_TAG" + exit 0 +fi + +log "pre-flight: sync version metadata" +run python3 scripts/sync_version.py + +if [[ "$DRY_RUN" != "1" ]] && { ! git diff --quiet HEAD -- || ! git diff --cached --quiet; }; then + die "tracked changes exist; commit them before a production deployment" +fi + +VERSION=$(python3 -c "import re; print(re.search(r'^version\s*=\s*\"([^\"]+)\"', open('pyproject.toml').read(), re.M).group(1))") +COMMIT=$(git rev-parse --short HEAD) +BUILD_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +IMAGE_TAG="${IMAGE_TAG:-${VERSION}-${COMMIT}}" +validate_tag "$IMAGE_TAG" + +log "release: version=$VERSION image=agenteval:$IMAGE_TAG commit=$COMMIT" + +log "pre-flight: remote production configuration" +run ssh "$HOST" "set -eu; mkdir -p '$REMOTE_DIR'; \ + test -f '$REMOTE_DIR/deploy/volcengine-102/.env' \ + || { echo 'missing production deploy/volcengine-102/.env' >&2; exit 1; }" + +log "sync committed release source → $HOST:$REMOTE_DIR" +run rsync -az --delete \ + --exclude='.git' \ + --exclude='.scratch' \ + --exclude='.env' \ + --exclude='node_modules' \ + --exclude='frontend/web/dist' \ + --exclude='.venv' \ + --exclude='__pycache__' \ + --exclude='.pytest_cache' \ + --exclude='.ruff_cache' \ + --exclude='data' \ + --exclude='config/config.json' \ + --exclude='*.db*' \ + ./ "$HOST:$REMOTE_DIR/" + +backup_remote_data "$IMAGE_TAG" + +log "build immutable production image agenteval:$IMAGE_TAG" +run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \ + IMAGE_TAG='$IMAGE_TAG' BUILD_COMMIT='$COMMIT' BUILD_TIME='$BUILD_TIME' \ + docker compose -f '$COMPOSE_FILE' build agenteval" + +log "start production services" +run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \ + IMAGE_TAG='$IMAGE_TAG' BUILD_COMMIT='$COMMIT' BUILD_TIME='$BUILD_TIME' \ + docker compose -f '$COMPOSE_FILE' up -d --remove-orphans" + +if [[ "$DRY_RUN" == "1" ]]; then + log "production deployment dry-run complete" + exit 0 +fi + +HEALTH_BODY=$(wait_for_health) +REMOTE_VERSION=$(printf '%s' "$HEALTH_BODY" | python3 -c "import json, sys; print(json.load(sys.stdin).get('version', ''))") +REMOTE_COMMIT=$(printf '%s' "$HEALTH_BODY" | python3 -c "import json, sys; print(json.load(sys.stdin).get('commit', ''))") +[[ "$REMOTE_VERSION" == "$VERSION" ]] || die "version mismatch: local=$VERSION remote=$REMOTE_VERSION" +[[ "$REMOTE_COMMIT" == "$COMMIT" ]] || die "commit mismatch: local=$COMMIT remote=$REMOTE_COMMIT" + +log "authenticated API smoke checks" +run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \ + API_KEY_VALUE=\$(sed -n 's/^AGENTEVAL_API_KEY=//p' deploy/volcengine-102/.env | head -1); \ + for API_PATH in /api/targets /api/scenarios /api/runs /api/campaigns /api/intelligent-evals /api/model-configs; do \ + HTTP_CODE=\$(curl -sS -o /dev/null -w '%{http_code}' -H \"X-API-Key: \$API_KEY_VALUE\" \"http://localhost:$APP_PORT\$API_PATH\"); \ + [ \"\$HTTP_CODE\" = '200' ] || { echo \"\$API_PATH returned \$HTTP_CODE\" >&2; exit 1; }; \ + echo \"\$API_PATH 200\"; \ + done" + +log "/api/health: $HEALTH_BODY" +log "production deploy OK: version=$VERSION image=agenteval:$IMAGE_TAG commit=$COMMIT" diff --git a/scripts/production-entrypoint.sh b/scripts/production-entrypoint.sh new file mode 100755 index 0000000..446506c --- /dev/null +++ b/scripts/production-entrypoint.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env sh +# Bootstrap legacy SQLite databases once, then apply real migrations on every +# subsequent start. This keeps the old pre-Alembic production database usable +# without stamping away future migrations. + +set -eu + +DB_PATH="${AGENTEVAL_DB_PATH:-data/agenteval.db}" + +if [ ! -f "$DB_PATH" ]; then + echo "production db not found; creating the initial SQLModel schema" + python -c "from agenteval.storage.db import init_db; init_db()" + alembic stamp head +else + HAS_ALEMBIC_VERSION=$(python -c "import sqlite3, sys; connection = sqlite3.connect(sys.argv[1]); row = connection.execute(\"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'alembic_version'\").fetchone(); connection.close(); print('yes' if row else 'no')" "$DB_PATH") + if [ "$HAS_ALEMBIC_VERSION" = "no" ]; then + echo "legacy production db detected; recording the current schema baseline" + python -c "from agenteval.storage.db import init_db; init_db()" + alembic stamp head + fi +fi + +alembic upgrade head +exec agenteval server start --host 0.0.0.0 --port 8000