208 lines
7.2 KiB
Bash
Executable File
208 lines
7.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Versioned production deployment for the volcengine-102 environment.
|
|
#
|
|
# Required:
|
|
# AGENTEVAL_PROD_HOST=<ssh host alias>
|
|
#
|
|
# Optional:
|
|
# AGENTEVAL_PROD_DIR=/opt/agenteval
|
|
# AGENTEVAL_PROD_PORT=8002
|
|
# AGENTEVAL_PROD_BACKUP_DIR=/var/backups/agenteval
|
|
#
|
|
# Usage:
|
|
# scripts/deploy-volcengine-102.sh --tag 0.8.0-6248568
|
|
# scripts/deploy-volcengine-102.sh --dry-run --tag 0.8.0-6248568
|
|
# scripts/deploy-volcengine-102.sh --rollback 0.8.0-6248568
|
|
|
|
set -euo pipefail
|
|
|
|
HOST="${AGENTEVAL_PROD_HOST:-}"
|
|
REMOTE_DIR="${AGENTEVAL_PROD_DIR:-/opt/agenteval}"
|
|
APP_PORT="${AGENTEVAL_PROD_PORT:-8002}"
|
|
BACKUP_DIR="${AGENTEVAL_PROD_BACKUP_DIR:-/var/backups/agenteval}"
|
|
COMPOSE_FILE="deploy/volcengine-102/docker-compose.yml"
|
|
|
|
DRY_RUN=0
|
|
IMAGE_TAG=""
|
|
ROLLBACK_TAG=""
|
|
|
|
usage() {
|
|
sed -n '2,18p' "$0"
|
|
}
|
|
|
|
log() { printf '\033[1;36m>> %s\033[0m\n' "$*"; }
|
|
warn() { printf '\033[1;33m!! %s\033[0m\n' "$*" >&2; }
|
|
die() { printf '\033[1;31m!! %s\033[0m\n' "$*" >&2; exit 1; }
|
|
run() {
|
|
if [[ "$DRY_RUN" == "1" ]]; then
|
|
printf '\033[2m[dry-run]'
|
|
printf ' %q' "$@"
|
|
printf '\033[0m\n'
|
|
else
|
|
"$@"
|
|
fi
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--tag)
|
|
[[ $# -ge 2 ]] || die "--tag requires a value"
|
|
IMAGE_TAG="$2"
|
|
shift 2
|
|
;;
|
|
--rollback)
|
|
[[ $# -ge 2 ]] || die "--rollback requires an image tag"
|
|
ROLLBACK_TAG="$2"
|
|
shift 2
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=1
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
die "unknown argument: $1"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
[[ -n "$HOST" ]] || die "AGENTEVAL_PROD_HOST is required"
|
|
[[ -z "$IMAGE_TAG" || -z "$ROLLBACK_TAG" ]] || die "--tag and --rollback cannot be used together"
|
|
|
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
RELEASE_DIR=""
|
|
cleanup_release_source() {
|
|
if [[ -n "$RELEASE_DIR" && -d "$RELEASE_DIR" ]]; then
|
|
rm -rf "$RELEASE_DIR"
|
|
fi
|
|
}
|
|
trap cleanup_release_source EXIT
|
|
|
|
validate_tag() {
|
|
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || die "invalid image tag: $1"
|
|
}
|
|
|
|
backup_remote_data() {
|
|
local release_tag="$1"
|
|
local timestamp backup_file
|
|
timestamp=$(date -u +"%Y%m%dT%H%M%SZ")
|
|
backup_file="agenteval-${release_tag}-${timestamp}.tgz"
|
|
|
|
log "backup production data volume → $BACKUP_DIR/$backup_file"
|
|
run ssh "$HOST" "set -eu; \
|
|
mkdir -p '$BACKUP_DIR'; \
|
|
if docker inspect agenteval >/dev/null 2>&1; then \
|
|
DATA_VOLUME=\$(docker inspect agenteval --format '{{range .Mounts}}{{if eq .Destination \"/app/data\"}}{{.Name}}{{end}}{{end}}'); \
|
|
[ -n \"\$DATA_VOLUME\" ] || { echo 'agenteval data volume not found' >&2; exit 1; }; \
|
|
docker run --rm -v \"\$DATA_VOLUME:/source:ro\" -v '$BACKUP_DIR:/backup' alpine:3.20 \
|
|
tar -czf '/backup/$backup_file' -C /source .; \
|
|
else \
|
|
echo 'first deployment: no existing agenteval container, backup skipped'; \
|
|
fi"
|
|
}
|
|
|
|
wait_for_health() {
|
|
local body=""
|
|
log "waiting for production /api/health" >&2
|
|
for _attempt in $(seq 1 30); do
|
|
body=$(ssh "$HOST" "curl -sf http://localhost:$APP_PORT/api/health 2>/dev/null" || true)
|
|
[[ -n "$body" ]] && break
|
|
sleep 2
|
|
done
|
|
[[ -n "$body" ]] || die "production health check did not respond after 60s"
|
|
printf '%s' "$body"
|
|
}
|
|
|
|
log "pre-flight: ssh connectivity"
|
|
run ssh -o ConnectTimeout=10 "$HOST" "echo ok >/dev/null" || die "cannot reach $HOST via ssh"
|
|
|
|
if [[ -n "$ROLLBACK_TAG" ]]; then
|
|
validate_tag "$ROLLBACK_TAG"
|
|
backup_remote_data "pre-rollback-$ROLLBACK_TAG"
|
|
log "rollback production image → agenteval:$ROLLBACK_TAG"
|
|
run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \
|
|
docker image inspect 'agenteval:$ROLLBACK_TAG' >/dev/null; \
|
|
IMAGE_TAG='$ROLLBACK_TAG' docker compose -f '$COMPOSE_FILE' up -d --no-build agenteval"
|
|
if [[ "$DRY_RUN" == "1" ]]; then
|
|
log "rollback dry-run complete"
|
|
exit 0
|
|
fi
|
|
HEALTH_BODY=$(wait_for_health)
|
|
log "/api/health: $HEALTH_BODY"
|
|
warn "database migrations are not downgraded; restore a matching backup only when the release requires it"
|
|
log "rollback OK: image=agenteval:$ROLLBACK_TAG"
|
|
exit 0
|
|
fi
|
|
|
|
log "pre-flight: sync version metadata"
|
|
run python3 scripts/sync_version.py
|
|
|
|
if [[ "$DRY_RUN" != "1" ]] && { ! git diff --quiet HEAD -- || ! git diff --cached --quiet; }; then
|
|
die "tracked changes exist; commit them before a production deployment"
|
|
fi
|
|
|
|
VERSION=$(python3 -c "import re; print(re.search(r'^version\s*=\s*\"([^\"]+)\"', open('pyproject.toml').read(), re.M).group(1))")
|
|
COMMIT=$(git rev-parse --short HEAD)
|
|
BUILD_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
IMAGE_TAG="${IMAGE_TAG:-${VERSION}-${COMMIT}}"
|
|
validate_tag "$IMAGE_TAG"
|
|
|
|
log "release: version=$VERSION image=agenteval:$IMAGE_TAG commit=$COMMIT"
|
|
|
|
log "pre-flight: remote production configuration"
|
|
run ssh "$HOST" "set -eu; mkdir -p '$REMOTE_DIR'; \
|
|
test -f '$REMOTE_DIR/deploy/volcengine-102/.env' \
|
|
|| { echo 'missing production deploy/volcengine-102/.env' >&2; exit 1; }"
|
|
|
|
log "prepare committed release source from HEAD"
|
|
RELEASE_DIR=$(mktemp -d "${TMPDIR:-/tmp}/agenteval-release.XXXXXX")
|
|
git archive --format=tar HEAD | tar -xf - -C "$RELEASE_DIR"
|
|
|
|
log "sync committed release source → $HOST:$REMOTE_DIR"
|
|
run rsync -az --delete \
|
|
--exclude='.env' \
|
|
--exclude='data' \
|
|
--exclude='config/config.json' \
|
|
--exclude='*.db*' \
|
|
"$RELEASE_DIR/" "$HOST:$REMOTE_DIR/"
|
|
|
|
backup_remote_data "$IMAGE_TAG"
|
|
|
|
log "build immutable production image agenteval:$IMAGE_TAG"
|
|
run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \
|
|
IMAGE_TAG='$IMAGE_TAG' BUILD_COMMIT='$COMMIT' BUILD_TIME='$BUILD_TIME' \
|
|
docker compose -f '$COMPOSE_FILE' build agenteval"
|
|
|
|
log "start production services"
|
|
run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \
|
|
IMAGE_TAG='$IMAGE_TAG' BUILD_COMMIT='$COMMIT' BUILD_TIME='$BUILD_TIME' \
|
|
docker compose -f '$COMPOSE_FILE' up -d --remove-orphans"
|
|
|
|
if [[ "$DRY_RUN" == "1" ]]; then
|
|
log "production deployment dry-run complete"
|
|
exit 0
|
|
fi
|
|
|
|
HEALTH_BODY=$(wait_for_health)
|
|
REMOTE_VERSION=$(printf '%s' "$HEALTH_BODY" | python3 -c "import json, sys; print(json.load(sys.stdin).get('version', ''))")
|
|
REMOTE_COMMIT=$(printf '%s' "$HEALTH_BODY" | python3 -c "import json, sys; print(json.load(sys.stdin).get('commit', ''))")
|
|
[[ "$REMOTE_VERSION" == "$VERSION" ]] || die "version mismatch: local=$VERSION remote=$REMOTE_VERSION"
|
|
[[ "$REMOTE_COMMIT" == "$COMMIT" ]] || die "commit mismatch: local=$COMMIT remote=$REMOTE_COMMIT"
|
|
|
|
log "authenticated API smoke checks"
|
|
run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \
|
|
API_KEY_VALUE=\$(sed -n 's/^AGENTEVAL_API_KEY=//p' deploy/volcengine-102/.env | head -1); \
|
|
for API_PATH in /api/targets /api/scenarios /api/runs /api/campaigns /api/intelligent-evals /api/model-configs; do \
|
|
HTTP_CODE=\$(curl -sS -o /dev/null -w '%{http_code}' -H \"X-API-Key: \$API_KEY_VALUE\" \"http://localhost:$APP_PORT\$API_PATH\"); \
|
|
[ \"\$HTTP_CODE\" = '200' ] || { echo \"\$API_PATH returned \$HTTP_CODE\" >&2; exit 1; }; \
|
|
echo \"\$API_PATH 200\"; \
|
|
done"
|
|
|
|
log "/api/health: $HEALTH_BODY"
|
|
log "production deploy OK: version=$VERSION image=agenteval:$IMAGE_TAG commit=$COMMIT"
|