ci: release the platform and the Java SDK together from a vX.Y.Z release

.github/workflows/java-sdk-release.yml → .github/workflows/release.yml

@@ -1,57 +1,67 @@
-name: Java SDK release
+name: Release
 
-# Publishes ai.intellistream:datahub-api-model and ai.intellistream:datahub-sdk to Maven Central,
-# in lockstep. Each release is signed on the release manager's own machine with their personal
-# key; this workflow never holds a signing key. It checks the signed bundle and uploads it.
+# Releases the platform and the Java SDK together, at one version, from a vX.Y.Z GitHub Release
+# on the minor version's release branch release/vX.Y:
+#   - the service jars and the Pulsar filter are attached to the GitHub Release, with SHA256SUMS
+#     and a build-provenance attestation;
+#   - ai.intellistream:datahub-api-model and ai.intellistream:datahub-sdk go to Maven Central.
 #
-# To release: set both versions in gradle.properties to X.Y.Z and merge; on that commit run
+# The SDK is signed on the release manager's own machine with their personal key; this workflow
+# never holds a signing key. To release X.Y.Z: on release/vX.Y, set `version` in gradle.properties
+# to X.Y.Z; on that commit run
 #   ./gradlew centralBundle -PsigningUseGpgCommand=true
-# then publish a GitHub Release tagged vX.Y.Z with build/central/datahub-central-X.Y.Z.zip attached.
-# Every published release runs this, so vX.Y.Z tags belong to the Java SDK.
+# then publish a GitHub Release tagged vX.Y.Z, targeting release/vX.Y, with
+# build/central/datahub-central-X.Y.Z.zip attached.
 on:
   release:
     types: [published]
-  # Rehearse the whole pipeline, with a throwaway signing key, when the version or the release
-  # machinery changes, so it is not first exercised during an actual release. The publish job
-  # skips unless this is a release.
+  # Rehearse everything but the upload and the attaching, with a throwaway signing key, when the
+  # version or the release machinery changes, so it is not first exercised during a release.
   pull_request:
     paths:
       - gradle.properties
       - build.gradle
-      - buildSrc/src/main/groovy/ai.intellistream.datahub.maven-central-conventions.gradle
-      - datahub-api-model/build.gradle
-      - datahub-java-sdk/build.gradle
+      - '*/build.gradle'
+      - buildSrc/**
       - datahub-java-sdk/RELEASE_SIGNERS
       - scripts/verify-central-bundle.sh
-      - .github/workflows/java-sdk-release.yml
+      - .github/workflows/release.yml
 
 permissions:
   contents: read
 
 jobs:
   versions:
-    name: Tag matches the versions
+    name: Tag matches the version and its release branch
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
+        with:
+          # The release-branch check needs the branches' history, not just the tagged commit.
+          fetch-depth: 0
       - run: |
           set -euo pipefail
-          model=$(grep -m1 '^apiModelVersion=' gradle.properties | cut -d= -f2)
-          sdk=$(grep -m1 '^javaSdkVersion=' gradle.properties | cut -d= -f2)
-          echo "apiModelVersion=$model javaSdkVersion=$sdk ref=${GITHUB_REF_NAME:-}"
-          # The SDK's POM pins api-model at the version built alongside it.
-          if [ "$model" != "$sdk" ]; then
-            echo "::error::apiModelVersion ($model) and javaSdkVersion ($sdk) disagree"
-            exit 1
-          fi
+          version=$(grep -m1 '^version=' gradle.properties | cut -d= -f2)
+          echo "version=$version ref=${GITHUB_REF_NAME:-}"
           # Only a release carries a version to check against; a rehearsal has none.
           if [ "${GITHUB_REF_TYPE:-}" = "tag" ]; then
             if ! printf '%s' "$GITHUB_REF_NAME" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
               echo "::error::'$GITHUB_REF_NAME' is not a release tag (expected vX.Y.Z)"
               exit 1
             fi
-            if [ "${GITHUB_REF_NAME#v}" != "$sdk" ]; then
-              echo "::error::tag $GITHUB_REF_NAME does not match javaSdkVersion $sdk"
+            if [ "${GITHUB_REF_NAME#v}" != "$version" ]; then
+              echo "::error::tag $GITHUB_REF_NAME does not match version $version in gradle.properties"
+              exit 1
+            fi
+            # Released from the minor version's release branch: the tagged commit has to be on
+            # it, not just anywhere in the repository.
+            branch=release/${GITHUB_REF_NAME%.*}
+            if ! git rev-parse --verify --quiet "refs/remotes/origin/$branch" >/dev/null; then
+              echo "::error::release branch $branch does not exist; tag vX.Y.Z on release/vX.Y"
+              exit 1
+            fi
+            if ! git merge-base --is-ancestor "$GITHUB_SHA" "origin/$branch"; then
+              echo "::error::$GITHUB_REF_NAME ($GITHUB_SHA) is not on $branch"
               exit 1
             fi
           fi
@@ -69,10 +79,43 @@ jobs:
           java-version: '25'
       - uses: gradle/actions/setup-gradle@v6
       # build.yml does not run on a release, so this is the only test run a release gets.
-      - run: ./gradlew :datahub-api-model:build :datahub-java-sdk:build
+      - run: ./gradlew build
+
+  platform:
+    name: Platform jars
+    runs-on: ubuntu-latest
+    needs: versions
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-java@v6
+        with:
+          distribution: temurin
+          java-version: '25'
+      - uses: gradle/actions/setup-gradle@v6
+      # The services the systemd examples run, one boot jar each, and the broker-side Pulsar
+      # filter. The NAR builds unversioned, because brokers load it by path; the release copy
+      # carries the version like the jars.
+      - run: |
+          set -euo pipefail
+          ./gradlew :datahub-api:bootJar :datahub-console:bootJar :datahub-stateless-consumer:bootJar \
+                    :datahub-analysis:bootJar :datahub-cleanup:bootJar :datahub-pulsar-filter:nar
+          version=$(grep -m1 '^version=' gradle.properties | cut -d= -f2)
+          mkdir dist
+          for m in api console stateless-consumer analysis cleanup; do
+            cp "datahub-$m/build/libs/datahub-$m-$version.jar" dist/
+          done
+          cp datahub-pulsar-filter/build/distributions/datahub-pulsar-filter.nar \
+             "dist/datahub-pulsar-filter-$version.nar"
+          (cd dist && sha256sum * > SHA256SUMS && cat SHA256SUMS)
+      - uses: actions/upload-artifact@v7
+        with:
+          name: platform-dist
+          retention-days: 7
+          path: dist/
 
-  verify:
-    name: Verify the signed bundle
+  sdk:
+    name: Verify the signed SDK bundle
     runs-on: ubuntu-latest
     needs: versions
     timeout-minutes: 30
@@ -92,9 +135,9 @@ jobs:
           else
             # A snapshot stages under timestamped file names that differ build to build, so the
             # rehearsal builds at the release version the snapshot is heading for.
-            v=$(grep -m1 '^javaSdkVersion=' gradle.properties | cut -d= -f2)
+            v=$(grep -m1 '^version=' gradle.properties | cut -d= -f2)
             echo "VERSION=${v%-SNAPSHOT}" >> "$GITHUB_ENV"
-            echo "VERSION_ARGS=-PjavaSdkVersion=${v%-SNAPSHOT} -PapiModelVersion=${v%-SNAPSHOT}" >> "$GITHUB_ENV"
+            echo "VERSION_ARGS=-Pversion=${v%-SNAPSHOT}" >> "$GITHUB_ENV"
           fi
       # What the tagged source builds to, unsigned. The bundle must match it byte for byte.
       - name: Build the expected artifacts
@@ -129,7 +172,7 @@ jobs:
           SIGNING_PASSWORD='' \
               ./gradlew centralBundle $VERSION_ARGS
           mkdir -p "$RUNNER_TEMP/bundle"
-          cp build/central/*.zip "$RUNNER_TEMP/bundle/"
+          cp "build/central/datahub-central-$VERSION.zip" "$RUNNER_TEMP/bundle/"
           fpr=$(gpg --list-keys --with-colons ci@example.invalid | awk -F: '/^fpr/{print $10; exit}')
           echo "$fpr  throwaway rehearsal key" > "$RUNNER_TEMP/signers"
           gpg --armor --export ci@example.invalid > "$RUNNER_TEMP/pubkeys.asc"
@@ -146,10 +189,10 @@ jobs:
           retention-days: 7
           path: ${{ runner.temp }}/bundle/*.zip
 
-  publish:
-    name: Publish to Maven Central
+  publish-sdk:
+    name: Publish the SDK to Maven Central
     runs-on: ubuntu-latest
-    needs: [build, verify]
+    needs: [build, platform, sdk]
     if: github.event_name == 'release' && github.repository_owner == 'IntelliStream-DataHub'
     environment: release
     timeout-minutes: 60
@@ -171,7 +214,7 @@ jobs:
           api=https://central.sonatype.com/api/v1/publisher
           id=$(curl -sS --fail-with-body -H "Authorization: Bearer $auth" \
                  -F "bundle=@datahub-central-$version.zip" \
-                 "$api/upload?name=datahub-java-sdk-$version&publishingType=AUTOMATIC")
+                 "$api/upload?name=datahub-$version&publishingType=AUTOMATIC")
           echo "deployment $id"
           # Validation plus the push to Central usually takes minutes; allow up to 45.
           for _ in $(seq 1 90); do
@@ -187,3 +230,28 @@ jobs:
           done
           echo "::error::deployment $id did not reach PUBLISHED in time; check the Portal"
           exit 1
+
+  attach-platform:
+    name: Attach the platform jars to the release
+    runs-on: ubuntu-latest
+    # After the SDK is on Central, so a release shows its jars only once all of it went out.
+    needs: publish-sdk
+    if: github.event_name == 'release' && github.repository_owner == 'IntelliStream-DataHub'
+    permissions:
+      contents: write
+      id-token: write
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@v7
+        with:
+          name: platform-dist
+          path: dist
+      # Signed provenance tying each file to this workflow run and the tagged commit;
+      # `gh attestation verify <file> --repo IntelliStream-DataHub/datahub-platform` checks it.
+      - uses: actions/attest@v4
+        with:
+          subject-path: 'dist/*'
+      - name: Attach
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh release upload "$GITHUB_REF_NAME" dist/* --repo "$GITHUB_REPOSITORY"

AGENTS.md

@@ -48,37 +48,48 @@ as the authoritative store, validation before anything goes async, one type labe
 on the frontend, calling datahub-api directly rather than extending the console's
 backend-for-frontend proxy. Read it before adding a feature that touches any of those.
 
-## Releasing the published artifacts
-
-`datahub-api-model` and `datahub-java-sdk` are the only modules published as Maven coordinates:
-`ai.intellistream:datahub-api-model` and `ai.intellistream:datahub-sdk` (the SDK's artifactId is
-**not** its Gradle project name). They are released in lockstep, because the SDK's POM pins the
-model at the exact version built alongside it. The shared POM, signing and staging setup is the
-`maven-central-conventions` plugin in `buildSrc`. `./gradlew centralBundle` stages both modules
-and zips the Maven Central deployment bundle. **A Central release can never be replaced or
-deleted**. A mistake can only be fixed by releasing a new version.
-
-Each release is signed on the release manager's own machine with their **personal** key; CI
-never holds a signing key. The keys allowed to sign are the primary fingerprints in
+## Releasing
+
+The platform and the Java SDK are released **together, at one version**: the `version` in the
+root `gradle.properties`, which Gradle applies to every module (override with `-Pversion=X.Y.Z`).
+A `vX.Y.Z` GitHub Release on the minor version's release branch `release/vX.Y` runs
+`.github/workflows/release.yml`, which publishes:
+
+- **the platform:** the five service boot jars (`datahub-api`, `-console`, `-stateless-consumer`,
+  `-analysis`, `-cleanup`) and the Pulsar filter (`datahub-pulsar-filter-X.Y.Z.nar`), attached
+  to the GitHub Release with a `SHA256SUMS` and a build-provenance attestation. The systemd
+  examples install these.
+- **the Java SDK:** `ai.intellistream:datahub-api-model` and `ai.intellistream:datahub-sdk` (the
+  SDK's artifactId is **not** its Gradle project name) on Maven Central. **A Central release can
+  never be replaced or deleted**. A mistake can only be fixed by releasing a new version.
+
+The SDK is signed on the release manager's own machine with their **personal** key; CI never
+holds a signing key. The keys allowed to sign are the primary fingerprints in
 `datahub-java-sdk/RELEASE_SIGNERS`, so adding a release manager is a reviewed change to that file.
+The shared POM, signing and staging setup is the `maven-central-conventions` plugin in
+`buildSrc`; `./gradlew centralBundle` stages both SDK modules and zips the Central bundle.
 
 To release `X.Y.Z`:
 
-1. Set both versions in the root `gradle.properties` (`apiModelVersion`, `javaSdkVersion`) to
-   `X.Y.Z` and merge.
-2. On that merged commit, with no `-P` version overrides, run
+1. On `release/vX.Y` (branched from `main` for a new minor, with fixes cherry-picked for a
+   patch), set `version` in `gradle.properties` to `X.Y.Z`.
+2. On that commit, with no `-P` version override, run
    `./gradlew centralBundle -PsigningUseGpgCommand=true`. It signs through your local gpg agent;
    add `-Psigning.gnupg.keyName=<fingerprint>` if you hold more than one key.
-3. `gh release create vX.Y.Z build/central/datahub-central-X.Y.Z.zip`. Every published GitHub
-   Release runs the release workflow, so `vX.Y.Z` tags and releases belong to the Java SDK.
-
-`.github/workflows/java-sdk-release.yml` then checks the tag against both versions, and builds and
-tests. `scripts/verify-central-bundle.sh` then verifies the attached bundle: it holds exactly the
-expected files, every file is signed by a listed key, and every file is byte-identical to what
-the tagged commit builds. The build is reproducible across JDK vendors, so a bundle built from any
-other commit or version fails. The job then waits in the `release` environment for approval, and
-uploads and publishes the verified bundle to Central. On a pull request that touches the release
-machinery, the workflow rehearses everything except the upload, signing with a throwaway key.
+3. `gh release create vX.Y.Z --target release/vX.Y build/central/datahub-central-X.Y.Z.zip`.
+
+The workflow:
+1. checks the tag against `version` and that the tagged commit is on `release/vX.Y`;
+2. builds and tests;
+3. builds the platform jars;
+4. verifies the attached SDK bundle with `scripts/verify-central-bundle.sh`. The bundle must hold
+   exactly the expected files, every file must be signed by a listed key, and every file must be
+   byte-identical to what the tagged commit builds; the build is reproducible across JDK vendors;
+5. waits in the `release` environment for approval, then publishes the SDK to Central;
+6. attaches the platform jars to the release.
+
+On a pull request that touches the version or the release machinery, it rehearses everything up
+to the upload, signing the SDK with a throwaway key.
 
 `centralBundle` refuses to build an unsigned bundle. Besides `-PsigningUseGpgCommand=true` it
 takes an in-memory key, `-PsigningKey`/`SIGNING_KEY` with `-PsigningPassword`, which is what the

build.gradle

@@ -91,7 +91,7 @@ tasks.register('centralBundle', Zip) {
     // Portal rejects a bundle carrying files it did not expect.
     exclude '**/maven-metadata*'
 
-    archiveFileName = "datahub-central-${providers.gradleProperty('javaSdkVersion').getOrElse('0.3.0-SNAPSHOT')}.zip"
+    archiveFileName = "datahub-central-${version}.zip"
     destinationDirectory = layout.buildDirectory.dir('central')
 
     // Central rejects an unsigned deployment, and the Sign tasks skip silently when no key

datahub-analysis/build.gradle

@@ -5,7 +5,6 @@ plugins {
 }
 
 group = 'ai.intellistream.datahub.analysis'
-version = '0.0.1-SNAPSHOT'
 
 configurations {
 	compileOnly {

datahub-api-model/build.gradle

@@ -76,7 +76,7 @@ jar { enabled = true; archiveClassifier = '' }
 // SDK. The group, licence/scm/developer POM metadata, sources+javadoc jars, the
 // LICENSE inside each jar, signing and the staging repository all come from the
 // maven-central-conventions plugin; `./gradlew centralBundle` builds the deployment.
-version = providers.gradleProperty('apiModelVersion').getOrElse('0.3.0-SNAPSHOT')
+// The version is the platform's, from the root gradle.properties.
 
 publishing.publications.mavenJava {
     pom {

datahub-api/build.gradle

@@ -9,7 +9,6 @@ plugins {
 }
 
 group = 'ai.intellistream.datahub.api'
-version = '0.0.1-SNAPSHOT'
 
 
 configurations {

datahub-cleanup/build.gradle

@@ -6,7 +6,6 @@ plugins {
 }
 
 group = 'ai.intellistream.datahub.cleanup'
-version = '0.0.1-SNAPSHOT'
 
 
 configurations {

datahub-console/build.gradle

@@ -5,7 +5,6 @@ plugins {
 }
 
 group = 'ai.intellistream.datahub'
-version = '0.0.1-SNAPSHOT'
 
 // Generates META-INF/build-info.properties, which Spring Boot turns into a BuildProperties bean.
 // The About dialog reports the version and build time from it (see AboutInfo). The Boot plugin

datahub-e2e/README.md

@@ -26,10 +26,10 @@ benchmark prints would be measuring that instead of the code.
 ./gradlew :datahub-api:bootJar :datahub-stateless-consumer:bootJar
 
 java -Xms4g -Xmx4g -XX:+UseG1GC -Dspring.profiles.active=dev,local \
-  -jar datahub-api/build/libs/datahub-api-0.0.1-SNAPSHOT.jar &
+  -jar datahub-api/build/libs/datahub-api-*-SNAPSHOT.jar &
 
 java -Xms4g -Xmx4g -XX:+UseG1GC -Dspring.profiles.active=dev,local \
-  -jar datahub-stateless-consumer/build/libs/datahub-stateless-consumer-0.0.1-SNAPSHOT.jar &
+  -jar datahub-stateless-consumer/build/libs/datahub-stateless-consumer-*-SNAPSHOT.jar &
 ```
 
 ## Environment

datahub-java-sdk/AGENTS.md

@@ -98,5 +98,5 @@ Thin, synchronous Java client for the DataHub Platform REST API, published as
 - Out-of-tree consumers install the artifact with `publishToMavenLocal`; there is no
   public Maven release yet. The remote `publish` repository exists only when
   `-PmavenPublishUrl` names one; `centralBundle` (root project) stages to a local directory
-  for Maven Central. The version comes from the `javaSdkVersion` Gradle property (default
-  `0.3.0-SNAPSHOT`), and the Maven artifactId is `datahub-sdk`, not the Gradle project name.
+  for Maven Central. The version is the platform's, the `version` in the root
+  `gradle.properties`, and the Maven artifactId is `datahub-sdk`, not the Gradle project name.

datahub-java-sdk/README.md

@@ -329,7 +329,7 @@ here and installing both artifacts to the local Maven repository:
 ```
 
 Then add `mavenLocal()` to that project's repositories and depend on
-`ai.intellistream:datahub-sdk:0.3.0-SNAPSHOT` (or whatever `javaSdkVersion` you built with).
+`ai.intellistream:datahub-sdk:1.0.0-SNAPSHOT` (or whatever `version` you built with).
 
 To also publish to a Maven repository of your own, pass its URL; there is deliberately no default
 (`centralBundle`, for Maven Central, stages to a local directory instead):

datahub-java-sdk/build.gradle

@@ -44,7 +44,7 @@ jar { enabled = true; archiveClassifier = '' }
 // model. The group, licence/scm/developer POM metadata, sources+javadoc jars, the
 // LICENSE inside each jar, signing and the staging repository all come from the
 // maven-central-conventions plugin; `./gradlew centralBundle` builds the deployment.
-version = providers.gradleProperty('javaSdkVersion').getOrElse('0.3.0-SNAPSHOT')
+// The version is the platform's, from the root gradle.properties.
 
 publishing.publications.mavenJava {
     // The Maven coordinate is datahub-sdk, not the Gradle project name datahub-java-sdk —

datahub-pulsar-filter/build.gradle

@@ -40,6 +40,8 @@ tasks.register('nar', Zip) {
     dependsOn tasks.named('jar')
     archiveExtension = 'nar'
     archiveClassifier = ''
+    // Unversioned, as brokers load it by path (README); a release attaches it with the version.
+    archiveVersion = ''
     from(tasks.named('jar')) {
         into 'META-INF/bundled-dependencies'
     }

datahub-rvm-converter/build.gradle

@@ -5,7 +5,6 @@ plugins {
 }
 
 group = 'ai.intellistream.datahub.rvm'
-version = '0.0.1-SNAPSHOT'
 
 configurations {
 	compileOnly {