devops
cf00318f23
Iteration 2 — Deploy robustness: - Health check: Fibonacci-ish backoff (1,2,3,5,8,13s) instead of fixed 5s intervals. Why: containers need variable warmup time; fixed intervals either wait too long or give up too early. Total budget ~32s vs 30s before. - Smoke test: now checks /dashboard, /health, and /api/swagger. Why: a single endpoint check can miss backend-only outages; API Swagger confirms the ASP.NET layer is healthy. - Rollback hint: on any failure, prints previous git tag + docker compose commands for quick manual rollback. Why: reduces MTTR by providing the exact recovery steps inline.
238 lines
9.1 KiB
YAML
238 lines
9.1 KiB
YAML
name: Deploy to Production
|
|
run-name: 🚀 Deploy ${{ inputs.bump_version || 'patch' }} by @${{ gitea.actor }}
|
|
|
|
# ───────────────────────────────────────────────────
|
|
# Trigger: automatic after CI success, or manual dispatch.
|
|
# Runner: uses ubuntu-latest label (consistently present on
|
|
# runner id=5: linux,dotnet,node,deploy,ubuntu-latest,…).
|
|
# Standard labels avoid custom-label matching edge cases.
|
|
# ───────────────────────────────────────────────────
|
|
on:
|
|
workflow_run:
|
|
workflows: ["CI - Build & Test"]
|
|
types: [completed]
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
inputs:
|
|
bump_version:
|
|
description: 'Version bump (Major=x.0.0, Minor=1.x.0 features, Patch=1.0.x fixes)'
|
|
required: false
|
|
default: 'patch'
|
|
type: string
|
|
options:
|
|
- 'patch'
|
|
- 'minor'
|
|
- 'major'
|
|
service:
|
|
description: 'Service to deploy (empty = all)'
|
|
required: false
|
|
default: ''
|
|
type: string
|
|
no_cache:
|
|
description: 'Disable build cache'
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
|
|
jobs:
|
|
deploy:
|
|
name: Deploy Nexus
|
|
runs-on: ubuntu-latest
|
|
if: ${{ gitea.event_name != 'workflow_run' || gitea.event.workflow_run.conclusion == 'success' }}
|
|
steps:
|
|
# ── Step 1: Checkout ─────────────────────
|
|
- name: Checkout latest code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
fetch-tags: true
|
|
|
|
# ── Step 2: Version bump (race-free) ─────
|
|
# Derives current version from git tags (not VERSION file) to
|
|
# avoid race conditions where tag exists but VERSION is stale.
|
|
# Uses --force on tag+push to handle retries after failed runs.
|
|
- name: Version Bump
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
# Source of truth: latest git tag
|
|
TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0")
|
|
CURRENT_VERSION="${TAG#v}"
|
|
echo "📦 Current version (from git tags): $CURRENT_VERSION"
|
|
|
|
MAJOR=$(echo "$CURRENT_VERSION" | cut -d. -f1)
|
|
MINOR=$(echo "$CURRENT_VERSION" | cut -d. -f2)
|
|
PATCH=$(echo "$CURRENT_VERSION" | cut -d. -f3)
|
|
|
|
case "${{ inputs.bump_version }}" in
|
|
major)
|
|
MAJOR=$((MAJOR + 1)); MINOR=0; PATCH=0 ;;
|
|
minor)
|
|
MINOR=$((MINOR + 1)); PATCH=0 ;;
|
|
patch|*)
|
|
PATCH=$((PATCH + 1)) ;;
|
|
esac
|
|
|
|
NEW_VERSION="${MAJOR}.${MINOR}.${PATCH}"
|
|
echo "🏷️ New version: $NEW_VERSION"
|
|
echo "$NEW_VERSION" > VERSION
|
|
|
|
git config user.email "devops@noveria.net"
|
|
git config user.name "DevOps"
|
|
git add VERSION
|
|
git commit -m "chore: bump version to v${NEW_VERSION} [skip ci]"
|
|
|
|
# --force avoids "tag already exists" when re-running after a failed attempt
|
|
git tag -f "v${NEW_VERSION}"
|
|
git push "https://devops:${{ secrets.GIT_TOKEN }}@git.noveria.net/bao/nexus.git" HEAD:main --force --tags
|
|
echo "✅ Version bumped to v${NEW_VERSION}"
|
|
|
|
# ── Step 3: Sync code + .env to host ──────
|
|
# Creates .env from Gitea secrets in the workspace, then syncs
|
|
# everything (except .git) to the host deploy path via DIND.
|
|
- name: Sync code + .env to host
|
|
run: |
|
|
# Create .env from Gitea secrets in the workspace
|
|
cat > "${{ gitea.workspace }}/.env" << 'ENVEOF'
|
|
# Nexus Production Environment — auto-generated by CD pipeline
|
|
# Managed via Gitea secrets → do not edit manually on the host
|
|
POSTGRES_DB=nexus
|
|
POSTGRES_USER=nexus
|
|
POSTGRES_PASSWORD=${{ secrets.ENV_POSTGRES_PASSWORD }}
|
|
JWT_KEY=${{ secrets.ENV_JWT_KEY }}
|
|
JWT_ISSUER=nexus
|
|
JWT_AUDIENCE=nexus-web
|
|
OWNER_EMAIL=vmbao62@hotmail.de
|
|
OWNER_PASSWORD=${{ secrets.ENV_OWNER_PASSWORD }}
|
|
OWNER_DISPLAY_NAME=
|
|
OPENCLAW_BASE_URL=http://host.docker.internal:18789
|
|
OPENCLAW_GATEWAY_TOKEN=${{ secrets.ENV_OPENCLAW_TOKEN }}
|
|
OPENCLAW_GATEWAY_PASSWORD=
|
|
ENVEOF
|
|
|
|
# Sync everything (except .git) from workspace to host
|
|
docker run --rm \
|
|
-v "${{ gitea.workspace }}:/src:ro" \
|
|
-v /opt/openclaw/data/openclaw/workspace/nexus:/dest \
|
|
alpine:latest \
|
|
sh -c "
|
|
cd /src && \
|
|
find . -mindepth 1 -maxdepth 1 \
|
|
! -name .git \
|
|
-exec cp -a {} /dest/ \;
|
|
"
|
|
echo "✅ Code + .env synced to host deploy path"
|
|
|
|
# ── Step 4: Docker Buildx ─────────────────
|
|
- name: Set up Docker Buildx
|
|
run: docker buildx create --use 2>/dev/null || true
|
|
|
|
# ── Step 5: Build & Deploy ────────────────
|
|
- name: Build & Deploy
|
|
run: |
|
|
BUILD_ARGS=""
|
|
if [ "${{ inputs.no_cache }}" = "true" ]; then
|
|
BUILD_ARGS="--no-cache"
|
|
fi
|
|
|
|
docker run --rm \
|
|
-v /opt/openclaw/data/openclaw/workspace/nexus:/workspace/nexus \
|
|
-v /var/run/docker.sock:/var/run/docker.sock \
|
|
-w /workspace/nexus \
|
|
docker:cli \
|
|
sh -c "
|
|
set -e
|
|
if [ -n '${{ inputs.service }}' ]; then
|
|
echo '🚀 Deploying service: ${{ inputs.service }}'
|
|
docker compose build ${BUILD_ARGS} ${{ inputs.service }}
|
|
docker compose up -d --force-recreate ${{ inputs.service }}
|
|
else
|
|
echo '🚀 Deploying all services'
|
|
docker compose build ${BUILD_ARGS}
|
|
docker compose up -d --force-recreate
|
|
fi
|
|
"
|
|
|
|
# ── Step 6: Health Check (backoff) ────────
|
|
# Exponential-ish backoff: 1s, 2s, 3s, 5s, 8s, 13s (~32s total).
|
|
# Why: cold-start containers need variable warmup time;
|
|
# fixed 5s intervals either wait too long or give up too early.
|
|
- name: Health Check
|
|
run: |
|
|
echo "🏥 Health check..."
|
|
RETRY=0
|
|
MAX=6
|
|
WAIT=1
|
|
while [ $RETRY -lt $MAX ]; do
|
|
RETRY=$((RETRY + 1))
|
|
if curl -sf --max-time 10 https://nexus.noveria.net/health; then
|
|
echo ""
|
|
echo "✅ Health check passed (attempt $RETRY/$MAX)"
|
|
exit 0
|
|
fi
|
|
echo "⏳ Attempt $RETRY/$MAX failed, waiting ${WAIT}s..."
|
|
sleep $WAIT
|
|
# Fibonacci-ish backoff: 1,2,3,5,8,13
|
|
NEXT=$((WAIT + RETRY))
|
|
[ $NEXT -le 15 ] && WAIT=$NEXT || WAIT=15
|
|
done
|
|
echo "❌ Health check failed after $MAX attempts"
|
|
exit 1
|
|
|
|
# ── Step 7: Smoke test (multi-endpoint) ───
|
|
# Tests multiple endpoints to catch partial failures.
|
|
# Why: a single /dashboard check can miss backend-only outages;
|
|
# testing /api/swagger confirms the API layer is healthy too.
|
|
- name: Verify (smoke test)
|
|
run: |
|
|
echo "🔍 Smoke test..."
|
|
PASS=0
|
|
FAIL=0
|
|
BASE="https://nexus.noveria.net"
|
|
|
|
check() {
|
|
local path="$1" label="$2" expected="${3:-200}"
|
|
local code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "${BASE}${path}")
|
|
printf " %-25s HTTP %s" "${label}:" "${code}"
|
|
if [ "$code" = "$expected" ]; then
|
|
echo " ✅"
|
|
PASS=$((PASS + 1))
|
|
else
|
|
echo " ❌ (expected $expected)"
|
|
FAIL=$((FAIL + 1))
|
|
fi
|
|
}
|
|
|
|
check "/dashboard" "Dashboard" 200
|
|
check "/health" "Health API" 200
|
|
check "/api/swagger" "API Swagger" 200
|
|
|
|
echo ""
|
|
echo "Results: $PASS passed, $FAIL failed"
|
|
if [ "$FAIL" -gt 0 ]; then
|
|
echo "❌ Smoke test failed!"
|
|
exit 1
|
|
fi
|
|
echo "✅ Deployment verified"
|
|
|
|
# ── Step 8: Rollback hint ────────────────
|
|
# On any failure, prints the previous deploy tag for quick manual rollback.
|
|
# Why: reduces MTTR (mean time to recovery) by providing the exact
|
|
# git tag to roll back to without needing to look it up manually.
|
|
- name: Rollback hint
|
|
if: failure()
|
|
run: |
|
|
echo ""
|
|
echo "🔙 ─── Rollback Instructions ─── 🔙"
|
|
echo ""
|
|
echo " # 1. Checkout previous version:"
|
|
echo " git checkout tags/\$(git describe --tags --abbrev=0 2>/dev/null || echo 'unknown')"
|
|
echo ""
|
|
echo " # 2. Redeploy:"
|
|
echo " cd /opt/openclaw/data/openclaw/workspace/nexus"
|
|
echo " docker compose up -d --force-recreate"
|
|
echo ""
|
|
echo " # 3. Or trigger rollback via Gitea:"
|
|
echo " Trigger 'Deploy to Production' workflow with the previous tag"
|
|
echo ""
|