ci: run bootstrap for real, and fix the two bugs it found

.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches: [main]
   pull_request:
+  workflow_dispatch:
 
 jobs:
   shellcheck:
@@ -18,15 +19,25 @@ jobs:
             -not -path './dotfiles/*' -print0 \
           | xargs -0 shellcheck -S warning -x
 
+  # Fast feedback: does bootstrap.sh survive an end-to-end --dry-run?
+  # It installs nothing, so it can only catch scripting errors — see
+  # bootstrap-real for the job that checks the install actually works.
   smoke:
     runs-on: ubuntu-latest
     container: ubuntu:24.04
     steps:
       - uses: actions/checkout@v4
       - name: Install prerequisites
         run: |
+          # The ubuntu image ships a dpkg config that drops /usr/share/doc/*,
+          # so packages install without the shell integration a real desktop
+          # gets (fzf's key-bindings.zsh, for one). Keep the container honest.
+          rm -f /etc/dpkg/dpkg.cfg.d/excludes
           apt-get update
-          apt-get install -y sudo curl wget gnupg lsb-release git stow ca-certificates
+          # zsh is pre-installed on purpose: --dry-run never installs anything,
+          # so without it every `command -v zsh` / require_cmd zsh branch in
+          # 20-shell.sh stays untested here (it broke CI twice that way).
+          apt-get install -y sudo curl wget gnupg lsb-release git stow ca-certificates zsh
           # create a non-root user (bootstrap refuses to run as root)
           useradd -m -s /bin/bash tester
           echo 'tester ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/tester
@@ -38,3 +49,38 @@ jobs:
       - name: Validate stow packages link cleanly (zsh/git/tmux)
         run: |
           su tester -c 'cd ~/forge && stow --no --target=$HOME --dir dotfiles zsh git tmux 2>&1 | tee /tmp/stow.log; ! grep -i conflict /tmp/stow.log'
+
+  # The real thing: no --dry-run. Slow (apt + several upstream downloads) and
+  # dependent on third-party endpoints, but it is the only job that proves the
+  # install works rather than merely that the scripts parse.
+  bootstrap-real:
+    runs-on: ubuntu-latest
+    container: ubuntu:24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create the unprivileged user
+        run: |
+          # See the smoke job: without this, /usr/share/doc/* never lands and
+          # the container stops resembling the machine we are provisioning.
+          rm -f /etc/dpkg/dpkg.cfg.d/excludes
+          apt-get update
+          # Bare minimum only: everything else must come from 00-preflight.sh,
+          # which is part of what we are testing.
+          apt-get install -y sudo ca-certificates curl git
+          useradd -m -s /bin/bash tester
+          echo 'tester ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/tester
+          cp -r . /home/tester/forge
+          chown -R tester:tester /home/tester/forge
+      - name: Run bootstrap for real (headless, no security)
+        run: |
+          su tester -c 'cd ~/forge && ./bootstrap.sh --yes --no-desktop --no-security'
+      - name: Assert the install actually landed
+        run: |
+          su tester -c 'cd ~/forge && bash ci/assert-bootstrap.sh'
+      - name: Re-run bootstrap (must be idempotent)
+        run: |
+          su tester -c 'cd ~/forge && ./bootstrap.sh --yes --no-desktop --no-security'
+      - name: Assert post-conditions still hold after the second run
+        run: |
+          su tester -c 'cd ~/forge && bash ci/assert-bootstrap.sh'

ci/assert-bootstrap.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# ci/assert-bootstrap.sh — post-conditions for a REAL bootstrap run.
+#
+# Run inside the CI container right after:
+#   ./bootstrap.sh --yes --no-desktop --no-security
+#
+# bootstrap.sh only proves it ran without erroring; this proves it *landed*:
+# every tool the install steps promise is on PATH, the zsh plugin checkouts
+# exist, the dotfiles are live symlinks into the repo (not silently backed up),
+# and the resulting .zshrc actually loads.
+#
+# Deliberately no `set -e`: we want the full list of what's missing in one CI
+# run, not just the first failure.
+set -uo pipefail
+
+FORGE_DIR="${FORGE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+# shellcheck source=../lib/common.sh
+source "$FORGE_DIR/lib/common.sh"
+
+# Same PATH the stowed .zshrc sets up: forge drops binaries in ~/.local/bin.
+export PATH="$HOME/bin:$HOME/.local/bin:$PATH"
+
+failures=0
+fail() { err "$*"; failures=$((failures + 1)); }
+
+check_cmd() {
+  if have "$1"; then
+    info "cmd   $1 → $(command -v "$1")"
+  else
+    fail "missing command: $1"
+  fi
+}
+
+check_dir() {
+  if [[ -d "$1" ]]; then
+    info "dir   $1"
+  else
+    fail "missing directory: $1"
+  fi
+}
+
+# A dotfile must resolve back into the repo. Anything else means stow skipped
+# it (conflict) or backed it up, leaving a dead copy: the config is not live.
+#
+# We test the resolved path, not `-L` on the leaf: stow folds trees, so a whole
+# package can be linked at a parent (~/.zsh -> ../forge/dotfiles/zsh/.zsh) and
+# the files inside it are then plain files reached through that symlink.
+check_link() {
+  local target="$1" dest
+  if [[ ! -e "$target" ]]; then
+    fail "missing (stow did not link it): $target"
+    return
+  fi
+  dest="$(readlink -f "$target")"
+  case "$dest" in
+    "$FORGE_DIR"/dotfiles/*) info "link  $target → $dest" ;;
+    *) fail "$target is not linked into the repo (resolves to: $dest)" ;;
+  esac
+}
+
+section "assert: commands on PATH"
+# Only tools the install steps install unconditionally. Optional ones
+# (aerc, tealdeer, jira-cli) are skipped: their absence is tolerated by design.
+for cmd in zsh tmux nvim rg fd bat jq tree htop fzf zoxide direnv \
+           eza gh glab lazygit starship mise stow git; do
+  check_cmd "$cmd"
+done
+
+section "assert: shell environment"
+check_dir "$HOME/.oh-my-zsh"
+check_dir "$HOME/.oh-my-zsh/custom/plugins/zsh-autosuggestions"
+check_dir "$HOME/.oh-my-zsh/custom/plugins/zsh-syntax-highlighting"
+check_dir "$HOME/.config/tmux/plugins/tpm"
+
+section "assert: dotfiles are stowed"
+check_link "$HOME/.zshrc"
+check_link "$HOME/.zsh/rc.d/10-aliases.zsh"
+check_link "$HOME/.zsh/functions/20-git.zsh"
+check_link "$HOME/.gitconfig"
+check_link "$HOME/.config/starship.toml"
+check_link "$HOME/.config/tmux/tmux.conf"
+
+section "assert: the zsh config loads"
+# -i so oh-my-zsh, the rc.d/ split, compinit, zoxide/direnv/mise/starship hooks
+# all get sourced; under a pty (script -qec) so it is a genuine interactive
+# shell — without one, fzf's key-bindings.zsh warns "can't change option: zle"
+# and drowns the real signal.
+#
+# The assertion is that startup prints *nothing*, not merely that it exits 0:
+# a `source` failing inside an oh-my-zsh plugin writes to stderr and leaves the
+# status at 0. timeout guards against a prompt sneaking in.
+startup_out="$(mktemp)"
+if ! TERM=xterm timeout 60 script -qec 'zsh -i -c true' /dev/null >"$startup_out" 2>&1; then
+  fail "interactive zsh exited non-zero"
+fi
+if [[ -s "$startup_out" ]]; then
+  fail "interactive zsh printed this at startup:"
+  cat "$startup_out" >&2
+else
+  ok "zsh -i starts silently"
+fi
+rm -f "$startup_out"
+
+section "assert: summary"
+if (( failures > 0 )); then
+  die "$failures assertion(s) failed"
+fi
+ok "bootstrap post-conditions verified"

install/10-packages.sh

@@ -47,13 +47,12 @@ fi
 if ! have glab; then
   log "Installing glab (GitLab CLI)"
   GLAB_VER="1.50.0"
-  case "$ARCH" in
-    amd64) GLAB_ARCH="x86_64" ;;
-    arm64) GLAB_ARCH="arm64" ;;
-    *) GLAB_ARCH="$ARCH" ;;
-  esac
+  # glab names its assets with Go arch strings (linux_amd64 / linux_arm64),
+  # which already match dpkg's — unlike lazygit and jira-cli below, which use
+  # x86_64. Translating amd64 -> x86_64 here yields a 404 on every 64-bit Intel
+  # box; only arm64 happened to work.
   tmp="$(mktemp -d)"
-  run bash -c "curl -fsSL 'https://gitlab.com/gitlab-org/cli/-/releases/v${GLAB_VER}/downloads/glab_${GLAB_VER}_linux_${GLAB_ARCH}.tar.gz' -o '$tmp/glab.tar.gz'"
+  run bash -c "curl -fsSL 'https://gitlab.com/gitlab-org/cli/-/releases/v${GLAB_VER}/downloads/glab_${GLAB_VER}_linux_${ARCH}.tar.gz' -o '$tmp/glab.tar.gz'"
   run tar -xzf "$tmp/glab.tar.gz" -C "$tmp"
   run install -m 0755 "$tmp/bin/glab" "$HOME/.local/bin/glab"
   rm -rf "$tmp"

install/50-stow.sh

@@ -10,6 +10,8 @@ require_cmd stow "run 00-preflight.sh."
 
 DOTFILES="$FORGE_DIR/dotfiles"
 [[ -d "$DOTFILES" ]] || die "missing $DOTFILES"
+# Canonical form, to compare against resolved symlink targets below.
+dotfiles_real="$(readlink -f "$DOTFILES")"
 
 # Desktop packages only stow when a GUI is present.
 PACKAGES=(zsh starship git tmux kitty aerc jira)
@@ -28,6 +30,13 @@ backup_conflicts() {
     rel="${f#"$pkgdir"/}"
     target="$HOME/$rel"
     if [[ -e "$target" && ! -L "$target" ]]; then
+      # stow folds trees: after a first run ~/.zsh is a symlink to
+      # $DOTFILES/zsh/.zsh, so $target resolves to the repo's *own* file —
+      # real, and not a symlink. Backing it up would rename the source out of
+      # the repo (it did: every re-run mangled dotfiles/ until CI caught it).
+      if [[ "$(readlink -f "$target")" == "$dotfiles_real"/* ]]; then
+        continue
+      fi
       backup_path "$target"
     fi
   done < <(find "$pkgdir" -type f -print0)