From 7c8f2b1e15a8eb93e1c2832a737a8d9e7b0dacd0 Mon Sep 17 00:00:00 2001 From: EOEboh Date: Tue, 28 Jul 2026 13:07:24 +0100 Subject: [PATCH] ci: add a manual deploy workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deploys were a laptop with Docker, the SSH key and a good connection. Nothing tied a deploy to a reviewed commit, and nothing surfaced drift — production ran a 12-day-old build against a current frontend without any signal, which is how the live checkout 500 happened. Actions tab → Deploy → Run workflow. Manual on purpose: merging should not ship to customers on its own, but a deploy should not depend on one machine either. It re-runs vet and the tests before building, because workflow_dispatch is not gated on the CI workflow and "it passed on the PR" is not the same as "it passes at the commit being deployed". The release is ordered so a failure cannot take the site down: the exact tag is pulled first, compose is only then repointed at an image the host already holds, and the container is restarted. If /health does not answer within 60s it restores the previous image and fails the run. The public endpoint is checked afterwards as well, since localhost being healthy says nothing about what customers reach. Images are tagged by commit as well as latest, so a rollback is re-running this against the previous commit rather than rebuilding it. scripts/deploy.sh stays as the break-glass path and now repoints compose back at the local image, since the workflow leaves it on a ghcr.io tag. It also asks before shipping and warns on a dirty tree. --- .github/workflows/deploy.yml | 153 +++++++++++++++++++++++++++++++++++ scripts/deploy.sh | 65 ++++++++++----- 2 files changed, 200 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..19d2e86 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,153 @@ +name: Deploy + +# Manual only. Deploys are a decision, not a side effect of merging — but they +# should not depend on one laptop having Docker, the SSH key and a good +# connection. Run this from the Actions tab. +on: + workflow_dispatch: + inputs: + ref: + description: "Branch, tag or SHA to deploy" + required: false + default: main + +concurrency: + # Two deploys racing onto one box leaves it in an unknown state. + group: deploy-production + cancel-in-progress: false + +env: + IMAGE: ghcr.io/${{ github.repository }} + +jobs: + deploy: + name: Build, push and release + runs-on: ubuntu-latest + environment: production + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Record what is being deployed + id: meta + run: | + echo "sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + echo "Deploying $(git rev-parse --short HEAD) — $(git log -1 --pretty=%s)" + + # Deploy only what passes. Actions does not gate workflow_dispatch on + # other workflows, so the checks are repeated here rather than assumed. + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Test before shipping + run: | + go vet ./... + go test -count=1 ./... + + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Tagged by commit as well as latest, so a rollback is a tag away rather + # than a rebuild of an older commit. + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 + push: true + tags: | + ${{ env.IMAGE }}:${{ steps.meta.outputs.sha }} + ${{ env.IMAGE }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Authorise SSH + run: | + mkdir -p ~/.ssh && chmod 700 ~/.ssh + printf '%s\n' "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + ssh-keyscan -H "${{ secrets.DEPLOY_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null + + # Pull the exact tag first, point compose at it, then restart. Compose + # never references an image the host does not already have, so a failed + # pull cannot take the site down. + - name: Release + env: + TAG: ${{ steps.meta.outputs.sha }} + run: | + ssh -o BatchMode=yes "${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}" \ + "IMAGE='${{ env.IMAGE }}' TAG='$TAG' bash -s" <<'REMOTE' + set -euo pipefail + cd /opt/hookdrop + + echo "→ pulling $IMAGE:$TAG" + docker pull "$IMAGE:$TAG" + + cp docker-compose.yml "docker-compose.yml.bak-$(date +%F-%H%M%S)" + PREVIOUS=$(grep -E '^\s*image:' docker-compose.yml | head -1 | sed 's/.*image:[[:space:]]*//') + echo "→ previous image: $PREVIOUS" + + # Clean shutdown checkpoints the SQLite WAL before the DB is snapshotted. + docker compose down + cp data/hookdrop.db "data/hookdrop.db.pre-deploy-$(date +%F-%H%M%S)" + + sed -i "s|^\(\s*image:\).*|\1 $IMAGE:$TAG|" docker-compose.yml + docker compose up -d + + echo "→ waiting for health" + for i in $(seq 1 20); do + if curl -fsS -m 5 http://localhost:8080/health >/dev/null 2>&1; then + echo "✓ healthy on attempt $i" + exit 0 + fi + sleep 3 + done + + echo "✗ unhealthy after 60s — rolling back to $PREVIOUS" + docker compose down + sed -i "s|^\(\s*image:\).*|\1 $PREVIOUS|" docker-compose.yml + docker compose up -d + docker logs hookdrop 2>&1 | tail -30 + exit 1 + REMOTE + + # Confirms the release from outside the box, not just from localhost. + - name: Verify from the public endpoint + run: | + for i in $(seq 1 10); do + body=$(curl -fsS -m 10 https://api.hookdrop.app/health) && { + echo "$body" + up=$(echo "$body" | sed 's/.*"uptime_seconds":\([0-9]*\).*/\1/') + if [ "$up" -lt 300 ]; then + echo "✓ uptime ${up}s — the new build is serving" + exit 0 + fi + echo "uptime ${up}s looks stale, retrying" + } + sleep 5 + done + echo "✗ public endpoint did not report a freshly started process" + exit 1 + + - name: Summary + if: always() + run: | + { + echo "### Deploy ${{ job.status }}" + echo "" + echo "- ref: \`${{ inputs.ref }}\`" + echo "- image: \`${{ env.IMAGE }}:${{ steps.meta.outputs.sha }}\`" + echo "" + echo "Roll back by re-running this workflow against the previous commit." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 93c6fb3..971212d 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,27 +1,56 @@ #!/bin/bash -set -e +# Break-glass deploy, straight from this machine. +# +# The normal route is the Deploy workflow in GitHub Actions (Actions tab → +# Deploy → Run workflow), which builds, pushes to GHCR, releases behind a +# health check and rolls back on failure. Use this only when Actions or GHCR +# is unavailable: it needs Docker locally, cross-compiles to amd64, and ships +# a ~13MB image over your connection. +# +# It also repoints docker-compose.yml at the locally loaded image, because the +# workflow leaves it pointing at a ghcr.io tag. +set -euo pipefail SERVER="deploy@178.104.166.5" -APP_DIR="/opt/hookdrop" -echo "→ Building Docker image..." +echo "⚠ Break-glass deploy — prefer the Deploy workflow in GitHub Actions." +echo " Working tree: $(git rev-parse --short HEAD)$([ -n "$(git status --porcelain)" ] && echo ' (DIRTY — uncommitted changes will ship)')" +read -r -p " Continue? [y/N] " reply +[ "$reply" = "y" ] || { echo "aborted"; exit 1; } + +echo "→ Building image (linux/amd64)..." docker build --platform linux/amd64 -t hookdrop:latest . -echo "→ Saving image..." -docker save hookdrop:latest | gzip > /tmp/hookdrop.tar.gz +echo "→ Uploading..." +docker save hookdrop:latest | gzip | ssh "$SERVER" 'cat > /tmp/hookdrop.tar.gz' + +echo "→ Releasing..." +ssh "$SERVER" 'bash -s' <<'REMOTE' +set -euo pipefail +cd /opt/hookdrop +docker load < /tmp/hookdrop.tar.gz + +cp docker-compose.yml "docker-compose.yml.bak-$(date +%F-%H%M%S)" +docker compose down +# Clean shutdown has checkpointed the WAL, so this snapshot is consistent. +cp data/hookdrop.db "data/hookdrop.db.pre-deploy-$(date +%F-%H%M%S)" + +# The workflow points this at a ghcr.io tag; send it back to the local image. +sed -i 's|^\(\s*image:\).*|\1 hookdrop:latest|' docker-compose.yml +docker compose up -d -echo "→ Uploading to server..." -scp /tmp/hookdrop.tar.gz $SERVER:/tmp/hookdrop.tar.gz +for i in $(seq 1 20); do + if curl -fsS -m 5 http://localhost:8080/health >/dev/null 2>&1; then + echo "✓ healthy on attempt $i" + rm -f /tmp/hookdrop.tar.gz + exit 0 + fi + sleep 3 +done -echo "→ Loading and restarting on server..." -ssh $SERVER << 'EOF' - docker load < /tmp/hookdrop.tar.gz - cd /opt/hookdrop - docker compose down - docker compose up -d - docker compose logs --tail=20 - rm /tmp/hookdrop.tar.gz -EOF +echo "✗ unhealthy after 60s" +docker logs hookdrop 2>&1 | tail -30 +exit 1 +REMOTE -rm /tmp/hookdrop.tar.gz -echo "✓ Deployed successfully" \ No newline at end of file +echo "✓ Deployed"