diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 3fb6fcd53..82512152a 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -5,10 +5,24 @@ on: branches: [main, dev] tags: ['v*'] workflow_dispatch: - -# Cancel in-progress runs for the same branch — saves CI minutes on rapid pushes + inputs: + tag: + description: >- + Rebuild an existing release tag (e.g. v0.9.1): checks out that tag's + source, builds amd64 + arm64 natively and re-pushes its version tags. + Empty = build the selected branch as usual. + type: string + default: '' + images: + description: Comma-separated images to build (backend, frontend, searxng, mcp-docs) or "all" + type: string + default: all + +# Cancel in-progress runs for the same branch — saves CI minutes on rapid pushes. +# A tag rebuild dispatched from a branch gets its own group so it neither +# cancels nor is cancelled by that branch's pushes. concurrency: - group: docker-build-${{ github.ref }} + group: docker-build-${{ github.ref }}-${{ inputs.tag }} cancel-in-progress: true jobs: @@ -21,292 +35,184 @@ jobs: frontend: ${{ steps.filter.outputs.frontend }} searxng: ${{ steps.filter.outputs.searxng }} mcp: ${{ steps.filter.outputs.mcp }} + ref: ${{ steps.release.outputs.ref }} + release_tag: ${{ steps.release.outputs.release_tag }} + latest: ${{ steps.release.outputs.latest }} steps: - name: Checkout uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 + # release_tag != '' switches the image jobs to native amd64 + arm64 + # builds merged into one index. A pushed v* tag builds itself; the + # `tag` dispatch input rebuilds an existing tag from any branch without + # moving it. A rebuild only claims `latest` when it is the newest stable + # tag, so rebuilding an old release cannot roll `latest` back. + - name: Resolve release + id: release + env: + INPUT_TAG: ${{ inputs.tag }} + REF: ${{ github.ref }} + REF_NAME: ${{ github.ref_name }} + run: | + set -euo pipefail + ref="" release_tag="" latest=auto + if [ -n "$INPUT_TAG" ]; then + if ! [[ "$INPUT_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$ ]]; then + echo "::error::tag input '$INPUT_TAG' is not a vX.Y.Z release tag" + exit 1 + fi + if ! git rev-parse -q --verify "refs/tags/$INPUT_TAG^{commit}" >/dev/null; then + echo "::error::tag $INPUT_TAG does not exist" + exit 1 + fi + newest=$(git tag -l 'v*' --sort=-v:refname | { grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' || true; } | sed -n 1p) + ref="refs/tags/$INPUT_TAG" release_tag="$INPUT_TAG" + if [ "$INPUT_TAG" = "$newest" ]; then latest=true; else latest=false; fi + elif [[ "$REF" == refs/tags/* ]]; then + release_tag="$REF_NAME" + fi + { + echo "ref=$ref" + echo "release_tag=$release_tag" + echo "latest=$latest" + } | tee -a "$GITHUB_OUTPUT" + - name: Classify changed images id: filter + env: + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + BEFORE: ${{ github.event.before }} + SHA: ${{ github.sha }} + INPUT_IMAGES: ${{ inputs.images }} run: | set -euo pipefail + declare -A build=([backend]=false [frontend]=false [searxng]=false [mcp]=false) + emit() { + for name in backend frontend searxng mcp; do + echo "$name=${build[$name]}" | tee -a "$GITHUB_OUTPUT" + done + } all_true() { - echo "backend=true" >> "$GITHUB_OUTPUT" - echo "frontend=true" >> "$GITHUB_OUTPUT" - echo "searxng=true" >> "$GITHUB_OUTPUT" - echo "mcp=true" >> "$GITHUB_OUTPUT" + for name in "${!build[@]}"; do build[$name]=true; done echo "Building all images." } - REF="${{ github.ref }}" - BEFORE="${{ github.event.before }}" ZERO="0000000000000000000000000000000000000000" - case "$REF" in - refs/tags/*) + if [ "$EVENT" = "workflow_dispatch" ]; then + if [ "${INPUT_IMAGES:-all}" = "all" ]; then all_true - exit 0 - ;; - esac - if [ "${{ github.event_name }}" != "push" ] || [ -z "$BEFORE" ] || [ "$BEFORE" = "$ZERO" ]; then + else + for image in ${INPUT_IMAGES//,/ }; do + case "$image" in + backend|frontend|searxng) build[$image]=true ;; + mcp-docs) build[mcp]=true ;; + *) echo "::error::unknown image '$image' in images input"; exit 1 ;; + esac + done + fi + emit + exit 0 + fi + if [[ "$REF" == refs/tags/* ]] || [ "$EVENT" != "push" ] || [ -z "$BEFORE" ] || [ "$BEFORE" = "$ZERO" ]; then all_true + emit exit 0 fi - if ! CHANGED=$(git diff --name-only "$BEFORE" "${{ github.sha }}"); then - echo "Could not diff $BEFORE..${{ github.sha }} — building all images." + if ! CHANGED=$(git diff --name-only "$BEFORE" "$SHA"); then + echo "Could not diff $BEFORE..$SHA — building all images." all_true + emit exit 0 fi flag() { - local name="$1" pattern="$2" - if echo "$CHANGED" | grep -qE "$pattern"; then - echo "$name=true" >> "$GITHUB_OUTPUT" - echo "$name=true" - else - echo "$name=false" >> "$GITHUB_OUTPUT" - echo "$name=false" - fi + if echo "$CHANGED" | grep -qE "$2"; then build[$1]=true; fi } - flag backend '^(backend/|packages/contracts/|package-lock\.json|package\.json$|\.dockerignore$|\.github/workflows/docker-build\.yml)' - flag frontend '^(frontend/|packages/contracts/|package-lock\.json|package\.json$|build-info\.json$|\.dockerignore$|\.github/workflows/docker-build\.yml)' - flag searxng '^(docker/searxng/|\.github/workflows/docker-build\.yml)' - flag mcp '^(mcp-docs/|\.dockerignore$|\.github/workflows/docker-build\.yml)' - + flag backend '^(backend/|packages/contracts/|package-lock\.json|package\.json$|\.dockerignore$|\.github/workflows/docker-(build|image)\.yml)' + flag frontend '^(frontend/|packages/contracts/|package-lock\.json|package\.json$|build-info\.json$|\.dockerignore$|\.github/workflows/docker-(build|image)\.yml)' + flag searxng '^(docker/searxng/|\.github/workflows/docker-(build|image)\.yml)' + flag mcp '^(mcp-docs/|\.dockerignore$|\.github/workflows/docker-(build|image)\.yml)' + emit + + # Each image job calls docker-image.yml: amd64-only on branch builds; + # native amd64 + arm64 runners merged into one index on release builds. docker-backend: name: Backend Image needs: changes if: needs.changes.outputs.backend == 'true' - runs-on: ubuntu-latest permissions: contents: read packages: write - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Log in to GHCR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 - with: - images: ghcr.io/compendiq/compendiq-ce-backend - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} - type=raw,value=dev,enable=${{ github.ref == 'refs/heads/dev' }} - - - name: Set up QEMU - if: startsWith(github.ref, 'refs/tags/') - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - - name: Build and push - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 - with: - context: . - file: ./backend/Dockerfile - platforms: ${{ startsWith(github.ref, 'refs/tags/') && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - push: true - cache-from: type=registry,ref=ghcr.io/compendiq/compendiq-ce-backend:buildcache - cache-to: type=registry,ref=ghcr.io/compendiq/compendiq-ce-backend:buildcache,mode=max - sbom: ${{ startsWith(github.ref, 'refs/tags/') }} - provenance: ${{ startsWith(github.ref, 'refs/tags/') && 'mode=max' || 'false' }} - - # Smoke-load the runtime image's prod node_modules. Guards against the - # class of bug where npm's hoister demotes a backend prod dep to - # `backend/node_modules/` — invisible to the runtime stage unless that - # tree is merged onto `/app/node_modules`. Builds succeed in that case; - # containers crash on boot. The historically-hoisted four (`p-limit`, - # `ipaddr.js`, two OpenTelemetry packages) are not sufficient: collab - # TipTap packages demote and `collab-schema.js` imports them on boot. - # amd64-only steps suffice; arm64 (tags) shares the same module graph. - # Uses the first tag from metadata, which is always present. - - name: Smoke-test runtime module resolution - if: ${{ !startsWith(github.ref, 'refs/tags/') }} - env: - IMAGE: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} - run: | - docker pull "$IMAGE" - docker run --rm --entrypoint node "$IMAGE" --input-type=module -e \ - "await import('p-limit'); await import('ipaddr.js'); await import('@opentelemetry/sdk-node'); await import('@opentelemetry/sdk-metrics'); await import('@tiptap/extension-code-block'); await import('@tiptap/extension-color'); await import('@tiptap/extension-text-style'); await import('@tiptap/y-tiptap'); await import('./dist/core/services/collab-schema.js'); console.log('runtime require smoke ok');" + uses: ./.github/workflows/docker-image.yml + with: + image: compendiq-ce-backend + context: . + file: ./backend/Dockerfile + ref: ${{ needs.changes.outputs.ref }} + release_tag: ${{ needs.changes.outputs.release_tag }} + latest: ${{ needs.changes.outputs.latest }} + attest: true + # Smoke-load the runtime image's prod node_modules on every built + # architecture. Guards against the class of bug where npm's hoister + # demotes a backend prod dep to `backend/node_modules/` — invisible to + # the runtime stage unless that tree is merged onto `/app/node_modules`. + # Builds succeed in that case; containers crash on boot. The + # historically-hoisted four (`p-limit`, `ipaddr.js`, two OpenTelemetry + # packages) are not sufficient: collab TipTap packages demote and + # `collab-schema.js` imports them on boot. + smoke_test: | + docker run --rm --entrypoint node "$IMAGE" --input-type=module -e \ + "await import('p-limit'); await import('ipaddr.js'); await import('@opentelemetry/sdk-node'); await import('@opentelemetry/sdk-metrics'); await import('@tiptap/extension-code-block'); await import('@tiptap/extension-color'); await import('@tiptap/extension-text-style'); await import('@tiptap/y-tiptap'); await import('./dist/core/services/collab-schema.js'); console.log('runtime require smoke ok');" docker-frontend: name: Frontend Image needs: changes if: needs.changes.outputs.frontend == 'true' - runs-on: ubuntu-latest permissions: contents: read packages: write - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Log in to GHCR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 - with: - images: ghcr.io/compendiq/compendiq-ce-frontend - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} - type=raw,value=dev,enable=${{ github.ref == 'refs/heads/dev' }} - - - name: Set up QEMU - if: startsWith(github.ref, 'refs/tags/') - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - # Stamp the committed placeholder build-info.json (commit: "unknown") - # with the real CE commit. The frontend bundle bakes __APP_COMMIT__ at - # build time from this file (frontend/vite.config.ts), and the EE flow - # never rebuilds the frontend — both editions ship this same CE image — - # so "Frontend commit" in Settings → System can only be filled in here. - # Mirrors scripts/build-enterprise.sh (git rev-parse --short HEAD) and - # the placeholder's JSON shape so vite's `ceCommit ?? commit` resolves it. - - name: Stamp build-info.json with the CE commit - run: | - SHORT_SHA="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)" - cat > build-info.json <> "$GITHUB_OUTPUT" + + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # `value=` is empty on branch builds, so the semver tags are skipped + # there exactly as before; on release builds it pins them to the tag + # even when the run was dispatched from a branch. + - name: Extract metadata + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 + with: + images: ${{ env.IMAGE }} + flavor: latest=${{ inputs.latest }} + labels: org.opencontainers.image.revision=${{ steps.src.outputs.sha }} + tags: | + type=semver,pattern={{version}},value=${{ inputs.release_tag }} + type=semver,pattern={{major}}.{{minor}},value=${{ inputs.release_tag }} + type=raw,value=latest,enable=${{ inputs.release_tag == '' && github.ref == 'refs/heads/main' }} + type=raw,value=dev,enable=${{ inputs.release_tag == '' && github.ref == 'refs/heads/dev' }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + # Stamp the committed placeholder build-info.json (commit: "unknown") + # with the real CE commit. The frontend bundle bakes __APP_COMMIT__ at + # build time from this file (frontend/vite.config.ts), and the EE flow + # never rebuilds the frontend — both editions ship this same CE image — + # so "Frontend commit" in Settings → System can only be filled in here. + # Mirrors scripts/build-enterprise.sh (git rev-parse --short HEAD) and + # the placeholder's JSON shape so vite's `ceCommit ?? commit` resolves it. + - name: Stamp build-info.json with the CE commit + if: inputs.stamp_build_info + run: | + SHORT_SHA="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)" + cat > build-info.json < Patch: dependency security updates — `@modelcontextprotocol/sdk` 1.31.0, OpenTelemetry spans without the DB username (attribute and HTTP metric names change, see below), `katex`, `sharp`, `shell-quote` and others. Node.js floor 22.13.0 for source installs. 20 commits since v0.9.0. diff --git a/CLAUDE.md b/CLAUDE.md index b4987aeea..b486f0918 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1369,6 +1369,8 @@ SemVer, pre-1.0. Single source of truth: **root `package.json` `"version"`**. Ba Feature PRs to `dev` → no bump. Release (`dev → main`) → bump all five `package.json`s (root, backend, frontend, packages/contracts, mcp-docs), merge, tag `vX.Y.Z`. Patch = bug, minor = feature or pre-1.0 breaking, major = `1.0.0` when production-ready. +Release images: the `v*` tag push runs `.github/workflows/docker-build.yml`, which builds each image natively per architecture (amd64 on `ubuntu-latest`, arm64 on `ubuntu-24.04-arm`; never QEMU — emulated arm64 `npm ci` hangs or dies with `Illegal instruction`), pushes by digest and merges one multi-arch index via the reusable `docker-image.yml`. Branch pushes (`dev`, `main`) stay amd64-only. To republish an existing release without moving its tag: `gh workflow run docker-build.yml --ref dev -f tag=vX.Y.Z -f images=all` (or e.g. `images=mcp-docs`); a rebuild only moves `latest` if it is the newest stable tag. + ## Dependencies - `npm install` from repo root only — workspaces require a single root lockfile. diff --git a/backend/src/dockerfile-prod-modules.test.ts b/backend/src/dockerfile-prod-modules.test.ts index 53c79e9dd..7897bea50 100644 --- a/backend/src/dockerfile-prod-modules.test.ts +++ b/backend/src/dockerfile-prod-modules.test.ts @@ -42,11 +42,11 @@ function runtimeStage(text: string): string { return match![0]; } +// The backend job passes its smoke script to the reusable docker-image.yml +// as the `smoke_test` input; capture that block up to the next job key. function smokeStep(workflow: string): string { - const match = workflow.match( - /name:\s*Smoke-test runtime module resolution[\s\S]*?(?=\n {2}[a-z]|\njobs:|$)/, - ); - expect(match, 'Smoke-test runtime module resolution step not found').not.toBeNull(); + const match = workflow.match(/\n\s+smoke_test:\s*\|[\s\S]*?(?=\n {2}[a-z]|$)/); + expect(match, 'backend smoke_test input not found').not.toBeNull(); return match![0]; } diff --git a/docs/architecture/05-deployment.md b/docs/architecture/05-deployment.md index d67b07f88..b82d356eb 100644 --- a/docs/architecture/05-deployment.md +++ b/docs/architecture/05-deployment.md @@ -168,7 +168,8 @@ That override is the only compose file that carries `build:` sections. Branch tags (`:dev`, `:latest`) are **linux/amd64 only** — the Docker -workflow publishes `linux/arm64` solely for `v*` release tags. The four +workflow publishes `linux/arm64` solely for `v*` release tags, built on +native arm64 runners and merged with the amd64 build into one index. The four Compendiq services (`frontend`, `backend`, `mcp-docs`, `searxng`) therefore set `platform: linux/amd64` so `docker compose pull` on Apple Silicon requests the amd64 manifest and runs it under Rosetta, instead of 404ing