[ci] Fix the empty Nightly release notes, and pin the build to the resolved core

.github/actions/get-latest-upstream/action.yml

@@ -51,7 +51,11 @@ runs:
           fi
         fi
         ASSET=$(curl -s -H "$AUTH_HEADER" https://api.github.com/repos/${AUTHORS}/selenium/releases/tags/${RELEASE} | jq -r '[.assets[] | select(.name | endswith(".jar"))] | last')
-        jar_file=$(echo "${ASSET}" | jq -r '.name')
+        jar_file=$(echo "${ASSET}" | jq -r '.name // empty')
+        if [ -z "${jar_file}" ]; then
+          echo "Release ${RELEASE} has no server jar asset"
+          exit 1
+        fi
         echo "Server package: ${jar_file}"
         VERSION=$(echo $jar_file | sed 's/selenium-server-//;s/\.jar//')
         # What that jar actually IS, not just what it is called. The nightly jar
@@ -60,13 +64,19 @@ runs:
         # version alone cannot tell two nightly cores apart. Anything keyed on the
         # core needs this, or it will treat last week's nightly as today's.
         ASSET_DIGEST=$(echo "${ASSET}" | jq -r '.digest // empty')
+        DIGEST_SOURCE="asset digest"
         if [ -z "${ASSET_DIGEST}" ] || [ "${ASSET_DIGEST}" = "null" ]; then
           # Older assets predate the digest field; identity then comes from the
           # asset id and the time it was last uploaded, which move together.
           ASSET_DIGEST=$(echo "${ASSET}" | jq -r '"\(.id)-\(.updated_at)"')
+          DIGEST_SOURCE="asset id and upload time (no digest field)"
         fi
         echo "BASE_RELEASE=${RELEASE} | BASE_VERSION=${VERSION} | VERSION=${VERSION}"
-        echo "Server package digest: ${ASSET_DIGEST}"
+        # Which of the two it is matters: only a real sha256 can be verified
+        # against the jar the build downloads, and if the digest field ever came
+        # back constant, everything keyed on the core would quietly go back to
+        # treating every nightly as the same one with nothing to notice it.
+        echo "Server package digest: ${ASSET_DIGEST} (from ${DIGEST_SOURCE})"
         echo "BASE_RELEASE=${RELEASE}" >> $GITHUB_ENV
         echo "BASE_VERSION=${VERSION}" >> $GITHUB_ENV
         echo "BASE_ASSET_DIGEST=${ASSET_DIGEST}" >> $GITHUB_ENV

.github/workflows/build-images.yml

@@ -53,6 +53,9 @@ on:
       base-release:
         description: 'Upstream release tag that core came from'
         value: ${{ jobs.decide.outputs.base-release }}
+      base-asset-digest:
+        description: 'sha256 of the server jar that core resolved to'
+        value: ${{ jobs.decide.outputs.base-asset-digest }}
       src-hash:
         description: 'Content hash the tag is keyed on'
         value: ${{ jobs.decide.outputs.src-hash }}
@@ -73,6 +76,7 @@ jobs:
       src-hash: ${{ steps.decide.outputs.src-hash }}
       base-version: ${{ steps.decide.outputs.base-version }}
       base-release: ${{ steps.decide.outputs.base-release }}
+      base-asset-digest: ${{ steps.decide.outputs.base-asset-digest }}
       registry: ${{ steps.decide.outputs.registry }}
     steps:
       - name: Checkout code
@@ -162,6 +166,9 @@ jobs:
           echo "src-hash=${SRC_HASH}" >> "$GITHUB_OUTPUT"
           echo "base-version=${BASE_VERSION}" >> "$GITHUB_OUTPUT"
           echo "base-release=${BASE_RELEASE}" >> "$GITHUB_OUTPUT"
+          # The build has to end up with the jar this hash was computed from, not
+          # merely with the same release tag - the tag's asset is replaced nightly.
+          echo "base-asset-digest=${BASE_ASSET_DIGEST}" >> "$GITHUB_OUTPUT"
           {
             echo "### Images"
             echo
@@ -246,15 +253,22 @@ jobs:
           command: make setup_dev_env
 
       # Use exactly the core decide hashed on, so the tag and the image agree.
-      # All three matter: Base/Dockerfile builds its download URL from the
-      # release tag AND the version
+      # All four matter. Base/Dockerfile builds its download URL from the release
+      # tag AND the version
       #   .../releases/download/${RELEASE}/selenium-server-${VERSION}.jar
       # so carrying only BASE_VERSION leaves BASE_RELEASE on the Makefile default
-      # and asks for a jar that does not exist.
+      # and asks for a jar that does not exist. And that URL names the asset, not
+      # its contents - the nightly one is replaced every night - so the digest is
+      # the only part of this that pins the build to the jar the decision was
+      # actually made about. Base/Dockerfile fails the build when what it
+      # downloads does not match, which also stops the two architecture jobs -
+      # they download independently, minutes apart - from putting different cores
+      # into one manifest.
       - name: Use the core decide resolved
         run: |
           echo "BASE_VERSION=${{ needs.decide.outputs.base-version }}" >> $GITHUB_ENV
           echo "BASE_RELEASE=${{ needs.decide.outputs.base-release }}" >> $GITHUB_ENV
+          echo "BASE_ASSET_DIGEST=${{ needs.decide.outputs.base-asset-digest }}" >> $GITHUB_ENV
           echo "VERSION=${{ needs.decide.outputs.base-version }}" >> $GITHUB_ENV
 
       # Native, single platform. Edge and Chrome for Testing are amd64-only and
@@ -275,7 +289,9 @@ jobs:
           timeout_minutes: 180
           max_attempts: 3
           retry_wait_seconds: 60
-          command: PLATFORMS="${PLATFORMS}" VERSION="${BASE_VERSION}" BUILD_DATE="${BUILD_DATE}" make build
+          command: |
+            PLATFORMS="${PLATFORMS}" VERSION="${BASE_VERSION}" BUILD_DATE="${BUILD_DATE}" \
+              BASE_ASSET_DIGEST="${BASE_ASSET_DIGEST}" make build
 
       # Also tag pr-<N>, so cleanup-pr-images.yml has something to delete when the
       # pull request closes. The src-* tag is shared and must outlive any one

.github/workflows/deploy.yml

@@ -347,8 +347,13 @@ jobs:
         with:
           github_token: ${{ secrets.SELENIUM_CI_TOKEN }}
           branch: trunk
+      # The release publishes <version>-<date>, which is what the notes read by
+      # default. GHCR_NAMESPACE has to be passed though: the mirror above is
+      # keyed on the repository owner, not on the script's SeleniumHQ default.
       - name: Create release notes (release_notes.md)
-        run: ./generate_release_notes.sh ${LATEST_TAG} origin/trunk ${GRID_VERSION} ${BUILD_DATE}
+        run: |
+          GHCR_NAMESPACE="ghcr.io/$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" \
+            ./generate_release_notes.sh ${LATEST_TAG} origin/trunk ${GRID_VERSION} ${BUILD_DATE}
       - name: Upload release notes
         uses: actions/upload-artifact@main
         with:

.github/workflows/k8s-scaling-test.yml

@@ -201,8 +201,14 @@ jobs:
           echo "AUTHORS=${AUTHORS}" >> $GITHUB_ENV
         env:
           AUTHORS: ${{ vars.AUTHORS || github.repository_owner }}
+      # A pull request from a fork gets no secrets, so these are empty and the
+      # login exits 1 - failing the job before a single test runs. The other test
+      # workflows that log in here already tolerate that; this one was missed,
+      # which is why a fork pull request saw only the scaling jobs go red. The
+      # login is an anonymous-pull-limit courtesy, not a requirement: every image
+      # the cluster needs is either built in this job or public.
       - name: Login Docker Hub
-        run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+        run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" || true
         env:
           DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
           DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}

.github/workflows/nightly.yml

@@ -51,6 +51,14 @@ jobs:
             make setup_dev_env
       - name: Output Docker info
         run: docker info
+      # The release notes read the arm64 browser versions by running the arm64
+      # image. On the build path the multi-architecture build registered the
+      # emulators as a side effect; the promotion path builds nothing, so nothing
+      # would.
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@main
+        with:
+          platforms: arm64
       - name: Set Selenium base version
         uses: ./.github/actions/get-latest-upstream
         with:
@@ -156,16 +164,28 @@ jobs:
           command: GHCR_NAMESPACE="ghcr.io/$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" VERSION="${GRID_VERSION}" BUILD_DATE=${BUILD_DATE} make release_ghcr_nightly
 #      - name: Update package versions
 #        run: make generate_nightly_sbom
+      # The changelog is the range <latest release tag>...trunk. An empty
+      # LATEST_TAG silently turns that into HEAD...trunk - nothing - so stop here
+      # rather than publishing notes with no changelog in them.
       - name: Get current latest tag
-        run: echo "LATEST_TAG=$(git describe --tags --abbrev=0 --exclude=nightly --exclude=selenium-grid*)" >> $GITHUB_ENV
+        run: |
+          set -euo pipefail
+          LATEST_TAG=$(git describe --tags --abbrev=0 --exclude=nightly --exclude='selenium-grid*')
+          echo "LATEST_TAG=${LATEST_TAG}" >> $GITHUB_ENV
       - name: Display latest tag
         run: echo ${LATEST_TAG}
       - name: Sets env var for nightly tag
         run: |
           echo "NEXT_TAG=nightly" >> $GITHUB_ENV
-          echo "FILTER_IMAGE_TAG=nightly" >> $GITHUB_ENV
+      # :nightly is the only tag the nightly publishes - the promotion never
+      # writes a <version>-<date> tag, and on that path nothing is in the local
+      # image store either. IMAGE_TAG is what the notes read the versions and the
+      # published manifests from.
       - name: Create release notes (release_notes.md)
-        run: ./generate_release_notes.sh ${LATEST_TAG} origin/trunk ${GRID_VERSION} ${BUILD_DATE}
+        run: |
+          IMAGE_TAG=nightly VIDEO_IMAGE_TAG=nightly \
+            GHCR_NAMESPACE="ghcr.io/$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" \
+            ./generate_release_notes.sh ${LATEST_TAG} origin/trunk ${GRID_VERSION} ${BUILD_DATE}
       - name: Set up Python
         uses: actions/setup-python@main
         with:

Base/Dockerfile

@@ -6,6 +6,13 @@
 # Arguments to define the version of dependencies to download
 ARG VERSION
 ARG RELEASE=selenium-${VERSION}
+# sha256 of the server jar this build is required to contain, as CI resolved it
+# before deciding to build. The download URL below is keyed on the release tag
+# and the version alone, and the nightly release keeps one asset name for a whole
+# snapshot cycle while its contents are replaced every night - so the URL is a
+# moving target and the two are not enough to say which core an image holds.
+# Empty on a local build, which takes whatever the URL currently serves.
+ARG SERVER_JAR_DIGEST=""
 # Default value should be aligned with upstream Selenium (https://github.com/SeleniumHQ/selenium/blob/trunk/MODULE.bazel)
 ARG OPENTELEMETRY_VERSION=1.65.0
 ARG GRPC_VERSION=1.83.1
@@ -26,6 +33,10 @@
 ARG TARGETARCH
 ARG TARGETVARIANT
 
+# Recorded so a published image can be traced back to the core it was built
+# against without pulling it apart, and inherited by every image built FROM base.
+LABEL org.seleniumhq.selenium.core.digest="${SERVER_JAR_DIGEST}"
+
 USER root
 
 ENV DEBIAN_FRONTEND=noninteractive \
@@ -130,6 +141,16 @@
   && chmod -R 775 /opt/selenium /var/run/supervisor /var/log/supervisor /etc/passwd ${HOME} ${VIDEO_FOLDER} \
   && wget --no-verbose https://github.com/${AUTHORS}/selenium/releases/download/${RELEASE}/selenium-server-${VERSION}.jar \
     -O /opt/selenium/selenium-server.jar \
+  # CI decides whether to build from the digest it resolved before the build
+  # started. If the jar moved in between, the image would be published under a
+  # tag naming a core it does not contain, and the two architecture builds -
+  # which download independently - could even end up with different cores in one
+  # manifest. Fail here instead. Anything that is not a sha256 (a local build, or
+  # an upstream asset that predates the digest field) skips the check and says so.
+  && case "${SERVER_JAR_DIGEST}" in \
+       sha256:*) echo "${SERVER_JAR_DIGEST#sha256:}  /opt/selenium/selenium-server.jar" | sha256sum -c - ;; \
+       *) echo "Server jar digest not verified (SERVER_JAR_DIGEST=${SERVER_JAR_DIGEST:-none})" ;; \
+     esac \
   && chgrp -R 0 /opt/selenium ${HOME} ${VIDEO_FOLDER} /opt/selenium/assets /var/run/supervisor /var/log/supervisor \
   && chmod -R g=u /opt/selenium ${HOME} ${VIDEO_FOLDER} /opt/selenium/assets /var/run/supervisor /var/log/supervisor \
   && setfacl -Rm u:${SEL_USER}:rwx /opt /opt/selenium ${HOME} ${VIDEO_FOLDER} /opt/selenium/assets /var/run/supervisor /var/log/supervisor \
@@ -209,34 +230,34 @@
 # Configure environement
 #======================================
     # Boolean value, maps "--bind-host"
 ENV SE_BIND_HOST="false" \
     SE_SERVER_PROTOCOL="http" \
     # Boolean value, maps "--reject-unsupported-caps"
     SE_REJECT_UNSUPPORTED_CAPS="false" \
     SE_DISTRIBUTOR_SLOT_SELECTOR="" \
     SE_OTEL_JAVA_GLOBAL_AUTOCONFIGURE_ENABLED="true" \
     SE_OTEL_TRACES_EXPORTER="otlp" \
     SE_SUPERVISORD_LOG_LEVEL="info" \
     SE_SUPERVISORD_CHILD_LOG_DIR="/tmp" \
     SE_SUPERVISORD_LOG_FILE="/tmp/supervisord.log" \
     SE_SUPERVISORD_PID_FILE="/tmp/supervisord.pid" \
     SE_SUPERVISORD_AUTO_RESTART="true" \
     SE_SUPERVISORD_START_RETRIES="5" \
     SE_SUPERVISORD_UNIX_SERVER_PASSWORD="secret" \
     SE_LOG_TIMESTAMP_FORMAT="%Y-%m-%d %H:%M:%S,%3N" \
     SE_LOG_LEVEL="INFO" \
     SE_HTTP_LOGS="false" \
     SE_STRUCTURED_LOGS="false" \
     SE_PLAIN_LOGS="true" \
     SE_ENABLE_TRACING="true" \
     SE_ENABLE_TLS="false" \
     SE_JAVA_OPTS_DEFAULT="" \
     SE_JAVA_HEAP_DUMP="false" \
     SE_JAVA_HTTPCLIENT_VERSION="HTTP_1_1" \
     SE_JAVA_SSL_TRUST_STORE="/opt/selenium/secrets/server.jks" \
     SE_JAVA_SSL_TRUST_STORE_PASSWORD="/opt/selenium/secrets/server.pass" \
     SE_JAVA_DISABLE_HOSTNAME_VERIFICATION="true" \
     SE_HTTPS_CERTIFICATE="/opt/selenium/secrets/tls.crt" \
     SE_HTTPS_PRIVATE_KEY="/opt/selenium/secrets/tls.key"
 
 CMD ["/opt/bin/entry_point.sh"]

Makefile

@@ -6,6 +6,11 @@ BASE_VERSION := $(or $(BASE_VERSION),$(BASE_VERSION),4.48.0)
 BINDING_VERSION := $(or $(BINDING_VERSION),$(BINDING_VERSION),4.48.0)
 BASE_RELEASE_NIGHTLY := $(or $(BASE_RELEASE_NIGHTLY),$(BASE_RELEASE_NIGHTLY),nightly)
 BASE_VERSION_NIGHTLY := $(or $(BASE_VERSION_NIGHTLY),$(BASE_VERSION_NIGHTLY),4.49.0-SNAPSHOT)
+# sha256 of the server jar the build must end up with, as get-latest-upstream
+# resolved it. Empty locally, where the build simply takes whatever the release
+# URL serves; CI passes it so a core that moved mid-build fails the build instead
+# of shipping under a tag that names the core it was supposed to have.
+BASE_ASSET_DIGEST := $(or $(BASE_ASSET_DIGEST),$(BASE_ASSET_DIGEST),)
 VERSION := $(or $(VERSION),$(VERSION),4.48.0)
 MVN_SELENIUM_VERSION := $(or $(MVN_SELENIUM_VERSION),$(MVN_SELENIUM_VERSION),latest.release)
 TAG_VERSION := $(VERSION)-$(BUILD_DATE)
@@ -90,6 +95,12 @@ SKIP_BUILD_TARGETS := $(or $(SKIP_BUILD_TARGETS),base hub distributor router ses
 	standalone_all_browsers standalone_docker standalone_kubernetes \
 	video ffmpeg keda_external_scaler update_go)
 
+# The same list, for the scripts that have to iterate it. generate_release_notes.sh
+# lists what a release published, and a second copy of these names in a shell
+# script is a copy that drifts.
+print_ci_images:
+	@echo $(CI_IMAGES)
+
 # Push what was just built, so the rest of the run can reuse it.
 # video does not carry the grid tag: it is built as
 # $(NAME)/video:$(FFMPEG_TAG_VERSION)-$(BUILD_DATE), and the compose files read it
@@ -488,6 +499,7 @@ gen_certs:
 
 base: update_go prepare_resources gen_certs
 	cd ./Base && SEL_PASSWD=$(SEL_PASSWD) docker buildx build --platform $(PLATFORMS) $(BUILD_ARGS) --build-arg VERSION=$(BASE_VERSION) --build-arg RELEASE=$(BASE_RELEASE) --build-arg AUTHORS=$(AUTHORS) \
+	--build-arg SERVER_JAR_DIGEST=$(BASE_ASSET_DIGEST) \
 	--secret id=SEL_PASSWD --sbom=true --attest type=provenance,mode=max -t $(NAME)/base:$(TAG_VERSION) .
 
 base_nightly: