#!/usr/bin/env bash # Versioned production deployment for the volcengine-102 environment. # # Required: # AGENTEVAL_PROD_HOST= # # Optional: # AGENTEVAL_PROD_DIR=/opt/agenteval # AGENTEVAL_PROD_PORT=8002 # AGENTEVAL_PROD_BACKUP_DIR=/var/backups/agenteval # # Usage: # scripts/deploy-volcengine-102.sh --tag 0.8.0-6248568 # scripts/deploy-volcengine-102.sh --dry-run --tag 0.8.0-6248568 # scripts/deploy-volcengine-102.sh --rollback 0.8.0-6248568 set -euo pipefail HOST="${AGENTEVAL_PROD_HOST:-}" REMOTE_DIR="${AGENTEVAL_PROD_DIR:-/opt/agenteval}" APP_PORT="${AGENTEVAL_PROD_PORT:-8002}" BACKUP_DIR="${AGENTEVAL_PROD_BACKUP_DIR:-/var/backups/agenteval}" COMPOSE_FILE="deploy/volcengine-102/docker-compose.yml" DRY_RUN=0 IMAGE_TAG="" ROLLBACK_TAG="" usage() { sed -n '2,18p' "$0" } log() { printf '\033[1;36m>> %s\033[0m\n' "$*"; } warn() { printf '\033[1;33m!! %s\033[0m\n' "$*" >&2; } die() { printf '\033[1;31m!! %s\033[0m\n' "$*" >&2; exit 1; } run() { if [[ "$DRY_RUN" == "1" ]]; then printf '\033[2m[dry-run]' printf ' %q' "$@" printf '\033[0m\n' else "$@" fi } while [[ $# -gt 0 ]]; do case "$1" in --tag) [[ $# -ge 2 ]] || die "--tag requires a value" IMAGE_TAG="$2" shift 2 ;; --rollback) [[ $# -ge 2 ]] || die "--rollback requires an image tag" ROLLBACK_TAG="$2" shift 2 ;; --dry-run) DRY_RUN=1 shift ;; -h|--help) usage exit 0 ;; *) die "unknown argument: $1" ;; esac done [[ -n "$HOST" ]] || die "AGENTEVAL_PROD_HOST is required" [[ -z "$IMAGE_TAG" || -z "$ROLLBACK_TAG" ]] || die "--tag and --rollback cannot be used together" ROOT="$(cd "$(dirname "$0")/.." && pwd)" cd "$ROOT" RELEASE_DIR="" cleanup_release_source() { if [[ -n "$RELEASE_DIR" && -d "$RELEASE_DIR" ]]; then rm -rf "$RELEASE_DIR" fi } trap cleanup_release_source EXIT validate_tag() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || die "invalid image tag: $1" } backup_remote_data() { local release_tag="$1" local timestamp backup_file timestamp=$(date -u +"%Y%m%dT%H%M%SZ") backup_file="agenteval-${release_tag}-${timestamp}.tgz" log "backup production data volume → $BACKUP_DIR/$backup_file" run ssh "$HOST" "set -eu; \ mkdir -p '$BACKUP_DIR'; \ if docker inspect agenteval >/dev/null 2>&1; then \ DATA_VOLUME=\$(docker inspect agenteval --format '{{range .Mounts}}{{if eq .Destination \"/app/data\"}}{{.Name}}{{end}}{{end}}'); \ [ -n \"\$DATA_VOLUME\" ] || { echo 'agenteval data volume not found' >&2; exit 1; }; \ docker run --rm -v \"\$DATA_VOLUME:/source:ro\" -v '$BACKUP_DIR:/backup' alpine:3.20 \ tar -czf '/backup/$backup_file' -C /source .; \ else \ echo 'first deployment: no existing agenteval container, backup skipped'; \ fi" } wait_for_health() { local body="" log "waiting for production /api/health" >&2 for _attempt in $(seq 1 30); do body=$(ssh "$HOST" "curl -sf http://localhost:$APP_PORT/api/health 2>/dev/null" || true) [[ -n "$body" ]] && break sleep 2 done [[ -n "$body" ]] || die "production health check did not respond after 60s" printf '%s' "$body" } log "pre-flight: ssh connectivity" run ssh -o ConnectTimeout=10 "$HOST" "echo ok >/dev/null" || die "cannot reach $HOST via ssh" if [[ -n "$ROLLBACK_TAG" ]]; then validate_tag "$ROLLBACK_TAG" backup_remote_data "pre-rollback-$ROLLBACK_TAG" log "rollback production image → agenteval:$ROLLBACK_TAG" run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \ docker image inspect 'agenteval:$ROLLBACK_TAG' >/dev/null; \ IMAGE_TAG='$ROLLBACK_TAG' docker compose -f '$COMPOSE_FILE' up -d --no-build agenteval" if [[ "$DRY_RUN" == "1" ]]; then log "rollback dry-run complete" exit 0 fi HEALTH_BODY=$(wait_for_health) log "/api/health: $HEALTH_BODY" warn "database migrations are not downgraded; restore a matching backup only when the release requires it" log "rollback OK: image=agenteval:$ROLLBACK_TAG" exit 0 fi log "pre-flight: sync version metadata" run python3 scripts/sync_version.py if [[ "$DRY_RUN" != "1" ]] && { ! git diff --quiet HEAD -- || ! git diff --cached --quiet; }; then die "tracked changes exist; commit them before a production deployment" fi VERSION=$(python3 -c "import re; print(re.search(r'^version\s*=\s*\"([^\"]+)\"', open('pyproject.toml').read(), re.M).group(1))") COMMIT=$(git rev-parse --short HEAD) BUILD_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") IMAGE_TAG="${IMAGE_TAG:-${VERSION}-${COMMIT}}" validate_tag "$IMAGE_TAG" log "release: version=$VERSION image=agenteval:$IMAGE_TAG commit=$COMMIT" log "pre-flight: remote production configuration" run ssh "$HOST" "set -eu; mkdir -p '$REMOTE_DIR'; \ test -f '$REMOTE_DIR/deploy/volcengine-102/.env' \ || { echo 'missing production deploy/volcengine-102/.env' >&2; exit 1; }" log "prepare committed release source from HEAD" RELEASE_DIR=$(mktemp -d "${TMPDIR:-/tmp}/agenteval-release.XXXXXX") git archive --format=tar HEAD | tar -xf - -C "$RELEASE_DIR" log "sync committed release source → $HOST:$REMOTE_DIR" run rsync -az --delete \ --exclude='.env' \ --exclude='data' \ --exclude='config/config.json' \ --exclude='*.db*' \ "$RELEASE_DIR/" "$HOST:$REMOTE_DIR/" backup_remote_data "$IMAGE_TAG" log "build immutable production image agenteval:$IMAGE_TAG" run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \ IMAGE_TAG='$IMAGE_TAG' BUILD_COMMIT='$COMMIT' BUILD_TIME='$BUILD_TIME' \ docker compose -f '$COMPOSE_FILE' build agenteval" log "start production services" run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \ IMAGE_TAG='$IMAGE_TAG' BUILD_COMMIT='$COMMIT' BUILD_TIME='$BUILD_TIME' \ docker compose -f '$COMPOSE_FILE' up -d --remove-orphans" if [[ "$DRY_RUN" == "1" ]]; then log "production deployment dry-run complete" exit 0 fi HEALTH_BODY=$(wait_for_health) REMOTE_VERSION=$(printf '%s' "$HEALTH_BODY" | python3 -c "import json, sys; print(json.load(sys.stdin).get('version', ''))") REMOTE_COMMIT=$(printf '%s' "$HEALTH_BODY" | python3 -c "import json, sys; print(json.load(sys.stdin).get('commit', ''))") [[ "$REMOTE_VERSION" == "$VERSION" ]] || die "version mismatch: local=$VERSION remote=$REMOTE_VERSION" [[ "$REMOTE_COMMIT" == "$COMMIT" ]] || die "commit mismatch: local=$COMMIT remote=$REMOTE_COMMIT" log "authenticated API smoke checks" run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \ API_KEY_VALUE=\$(sed -n 's/^AGENTEVAL_API_KEY=//p' deploy/volcengine-102/.env | head -1); \ for API_PATH in /api/targets /api/scenarios /api/runs /api/campaigns /api/intelligent-evals /api/model-configs; do \ HTTP_CODE=\$(curl -sS -o /dev/null -w '%{http_code}' -H \"X-API-Key: \$API_KEY_VALUE\" \"http://localhost:$APP_PORT\$API_PATH\"); \ [ \"\$HTTP_CODE\" = '200' ] || { echo \"\$API_PATH returned \$HTTP_CODE\" >&2; exit 1; }; \ echo \"\$API_PATH 200\"; \ done" log "/api/health: $HEALTH_BODY" log "production deploy OK: version=$VERSION image=agenteval:$IMAGE_TAG commit=$COMMIT"