Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions .claude/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,16 @@ property-only Maven profiles. Released artifacts encode the Spark version in the
- **Always `mvn clean` when switching Spark profiles**: the ANTLR-generated parser sources are
not regenerated on profile switch, and 4.9.3-generated code fails at runtime against the
4.13.1 runtime ("Could not deserialize ATN with version 3").
- **Release**: pushing a `v*` tag runs `.github/workflows/release.yml` — builds tantivy4java
(linux-x86_64) once, then publishes each Spark leg to Maven Central and attaches shaded jars
to a GitHub Release. For local builds of all three legs:
- **Release**: run `.github/workflows/release.yml` from `main` with an existing `v*` tag
(Actions → Release → Run workflow). `dry-run` is ticked by default and publishes nothing;
untick it to publish. Pushing a tag does not start a release. The artifact names carry
`spark.version` from `pom.xml`; when that differs from the last release, a publish run asks
for the `expected-spark-versions` input (the dry run prints the value). The run builds tantivy4java
(linux-x86_64) from source at the commits pinned in
`.github/scripts/release/native-pins.txt` (add a line there when bumping tantivy4java in
`pom.xml`), builds each Spark leg unsigned, and only then signs and uploads from a separate
job that runs no project code. Runbook: `.github/scripts/release/README.md`. For local
builds of all three legs:
`dev/build-release.sh <base-version>` (shaded jars land in `dist/`, platform classifier
auto-detected for the current machine).

Expand Down
387 changes: 387 additions & 0 deletions .github/scripts/release/README.md

Large diffs are not rendered by default.

215 changes: 215 additions & 0 deletions .github/scripts/release/assemble.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
#!/usr/bin/env bash
# Turn the build jobs' outputs into the exact trees that will be signed and
# uploaded, after checking them. Runs no project code: it only reads files.
#
# Everything in IN was produced by jobs that ran the project's build, so it is
# treated as untrusted until checked. What comes out is an allow-list result:
# only the expected coordinates, only the expected file names.
#
# Environment:
# IN directory with the downloaded artifacts:
# native/ build-native.sh output
# artifacts-<profile>/ build-artifacts.sh output, per profile
# OUT staging directory to create:
# versions.txt the versions, one per line
# release-notes.md the "Build inputs" block
# bundles/<version>/ Maven repository layout for that
# version: the six files and their
# md5/sha1/sha256/sha512 (no signatures)
# MANIFEST.sha256 SHA-256 of every file above
# TAG, COMMIT, BASE_VERSION, EXPECTED_VERSIONS from plan.sh
# RUN_URL link to this workflow run (optional, for the notes)
#
# Step outputs: manifest_sha256 (digest of MANIFEST.sha256), versions.
set -euo pipefail
here="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=lib.sh
. "$here/lib.sh"

: "${IN:?}" "${OUT:?}" "${TAG:?}" "${COMMIT:?}" "${BASE_VERSION:?}" "${EXPECTED_VERSIONS:?}"
need unzip python3
[[ "$TAG" =~ $TAG_RE ]] || die "'$TAG' is not a release tag"
[ "v$BASE_VERSION" = "$TAG" ] || die "base version $BASE_VERSION does not belong to tag $TAG"
[[ "$COMMIT" =~ $SHA_RE ]] || die "'$COMMIT' is not a commit id"

# check_pom <pom> <version>: coordinates must be the ones this repository
# publishes, and the metadata Maven Central requires must be present.
check_pom() {
python3 - "$1" "$GROUP_ID" "$ARTIFACT_ID" "$2" << 'PY' || die "$(basename "$1") failed the pom checks (above)"
import sys
import xml.etree.ElementTree as ET

path, group, artifact, version = sys.argv[1:5]
ns = "{http://maven.apache.org/POM/4.0.0}"
try:
root = ET.parse(path).getroot()
except ET.ParseError as exc:
sys.exit("pom is not well-formed XML: %s" % exc)

def text(*names):
node = root
for name in names:
node = node.find(ns + name) if node is not None else None
return (node.text or "").strip() if node is not None else ""

problems = []
if root.tag != ns + "project":
problems.append("root element is not a Maven <project>")
if root.find(ns + "parent") is not None:
problems.append("has a <parent>; coordinates must be stated in the pom itself")
for names, want in ((("groupId",), group), (("artifactId",), artifact), (("version",), version)):
got = text(*names)
if got != want:
problems.append("%s is '%s', expected '%s'" % ("/".join(names), got, want))
if text("packaging") not in ("", "jar"):
problems.append("packaging is '%s', expected jar" % text("packaging"))
required = (("name",), ("description",), ("url",),
("licenses", "license", "name"), ("licenses", "license", "url"),
("developers", "developer", "name"),
("scm", "connection"), ("scm", "url"))
for names in required:
if not text(*names):
problems.append("missing %s (required by Maven Central)" % "/".join(names))
if problems:
sys.exit("\n".join(problems))
PY
}

# --- the native build ---------------------------------------------------------
native="$IN/native"
verify_sums "$native"
ninfo="$native/native-inputs.txt"
t4j_version="$(kv "$ninfo" tantivy4java_version)"
[[ "$t4j_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "the native artifact does not state a tantivy4java version"
[ "$(kv "$ninfo" cargo_lock_unchanged)" = true ] || die "the native build does not record an unchanged Cargo.lock"
t4j_jar="$native/repository/tantivy4java-$t4j_version-$NATIVE_CLASSIFIER.jar"
[ -f "$t4j_jar" ] || die "the native artifact has no $(basename "$t4j_jar")"
t4j_sha="$(hash_of sha256 "$t4j_jar")"
libs="$(unzip -Z1 "$t4j_jar" | grep -E '\.so$' || true)"
[ -n "$libs" ] || die "$(basename "$t4j_jar") contains no native library (*.so)"

rm -rf "$OUT"
mkdir -p "$OUT/bundles"
: > "$OUT/versions.txt"

# --- each Spark profile -------------------------------------------------------
for profile in $PROFILES; do
d="$IN/artifacts-$profile"
[ -d "$d" ] || die "no build output for $profile"
verify_sums "$d"
info="$d/build-info.txt"
[ "$(kv "$info" profile)" = "$profile" ] || die "artifacts-$profile was not built with profile $profile"
version="$(kv "$info" version)"
check_version "$BASE_VERSION" "$profile" "$version"
[ "$version" = "$(profile_version "$EXPECTED_VERSIONS" "$profile")" ] \
|| die "artifacts-$profile holds $version, which is not the version the plan announced for $profile"
[ "${BASE_VERSION}_spark_$(kv "$info" spark_version)" = "$version" ] \
|| die "artifacts-$profile: version $version does not match its recorded Spark version"

expected="$({ artifact_names "$version" | sed 's|^|files/|'; echo build-info.txt; } | LC_ALL=C sort)"
actual="$(cd "$d" && find . -type f ! -path ./SHA256SUMS | sed 's|^\./||' | LC_ALL=C sort)"
[ "$expected" = "$actual" ] \
|| die "artifacts-$profile does not hold exactly the expected files. Expected: $(echo $expected). Found: $(echo $actual)"

[ "$(kv "$info" tantivy4java_jar_sha256)" = "$t4j_sha" ] \
|| die "artifacts-$profile was not built with this run's tantivy4java jar"

p="$ARTIFACT_ID-$version"
check_pom "$d/files/$p.pom" "$version"
for name in $(artifact_names "$version"); do
case "$name" in
*.jar) unzip -tqq "$d/files/$name" > /dev/null 2>&1 || die "$name is not a readable jar" ;;
esac
done

# The native library inside the published jars must be, byte for byte, the
# one this run built from the pinned commits.
for jar in "$p-$NATIVE_CLASSIFIER-shaded.jar" "$p-jar-with-dependencies.jar"; do
while IFS= read -r lib; do
want="$(unzip -p "$t4j_jar" "$lib" | hash_stream sha256)"
got="$(unzip -p "$d/files/$jar" "$lib" 2> /dev/null | hash_stream sha256 || true)"
[ "$got" = "$want" ] || die "$jar does not contain this run's $lib"
done <<< "$libs"
done

vd="$OUT/bundles/$version/$GROUP_PATH/$version"
mkdir -p "$vd"
for name in $(artifact_names "$version"); do
cp "$d/files/$name" "$vd/$name"
for algo in $CHECKSUM_ALGOS; do
printf '%s' "$(hash_of "$algo" "$vd/$name")" > "$vd/$name.$algo"
done
done
echo "$version" >> "$OUT/versions.txt"
done

# --- notes ----------------------------------------------------------------------
# The values below were written by the native job, which runs build code, and
# they end up in the release notes. Identifiers must have their exact form;
# free-text tool versions are shown only if they are plain text.
SAFE_RE='^[A-Za-z0-9 ._,:/@()+="-]*$'
safe() {
if [ "${#1}" -le 200 ] && [[ "$1" =~ $SAFE_RE ]]; then printf '%s' "$1"; else printf '(not shown: unexpected characters)'; fi
}
[[ "$(kv "$ninfo" tantivy4java_tag)" == "v$t4j_version" ]] || die "the native artifact's tag is not v$t4j_version"
[[ "$(kv "$ninfo" tantivy4java_commit)" =~ $SHA_RE ]] || die "the native artifact does not state a tantivy4java commit"
for key in quickwit_commit tantivy_commit; do
value="$(kv "$ninfo" "$key")"
[ "$value" = "-" ] || [[ "$value" =~ $SHA_RE ]] || die "the native artifact's $key is not a commit id"
done
[[ "$(kv "$ninfo" cargo_lock_sha256)" =~ ^[0-9a-f]{64}$ ]] || die "the native artifact does not state a Cargo.lock digest"
{
echo "<!-- release-build-inputs:start -->"
echo "### Build inputs"
echo
if [ -n "${RUN_URL:-}" ]; then
echo "Built from commit \`$COMMIT\` (tag \`$TAG\`) by [this workflow run]($RUN_URL)."
else
echo "Built from commit \`$COMMIT\` (tag \`$TAG\`)."
fi
echo "The native library was compiled in that run from the commits below; nothing was restored from a cache."
echo
echo "| Input | Value |"
echo "|---|---|"
echo "| tantivy4java | \`$(kv "$ninfo" tantivy4java_tag)\` at \`$(kv "$ninfo" tantivy4java_commit)\` |"
echo "| quickwit fork | \`$(kv "$ninfo" quickwit_commit)\` |"
echo "| tantivy fork | \`$(kv "$ninfo" tantivy_commit)\` |"
other="$(kv "$ninfo" other_git_dependencies)"
[ -z "$other" ] || echo "| other git dependencies | \`$(safe "$other")\` |"
echo "| Cargo.lock | SHA-256 \`$(kv "$ninfo" cargo_lock_sha256)\`, unchanged by the build |"
for key in rustc cargo protoc java maven runner_image; do
value="$(kv "$ninfo" "$key")"
[ -z "$value" ] || echo "| ${key//_/ } | \`$(safe "$value")\` |"
done
echo
echo "| Version | SHA-256 of \`-$NATIVE_CLASSIFIER-shaded.jar\` |"
echo "|---|---|"
while IFS= read -r version; do
f="$OUT/bundles/$version/$GROUP_PATH/$version/$ARTIFACT_ID-$version-$NATIVE_CLASSIFIER-shaded.jar"
echo "| \`$version\` | \`$(hash_of sha256 "$f")\` |"
done < "$OUT/versions.txt"
echo "<!-- release-build-inputs:end -->"
} > "$OUT/release-notes.md"

write_sums "$OUT" MANIFEST.sha256
manifest_sha="$(hash_of sha256 "$OUT/MANIFEST.sha256")"
output manifest_sha256 "$manifest_sha"
output versions "$(tr '\n' ' ' < "$OUT/versions.txt" | sed 's/ $//')"

{
echo "## Staged for $TAG"
echo
echo "Manifest SHA-256: \`$manifest_sha\`. The publish job signs and uploads exactly the files this manifest lists, and refuses anything else."
echo
sed -e '/^<!-- release-build-inputs/d' "$OUT/release-notes.md"
echo
echo "<details><summary>Every staged file</summary>"
echo
echo "| File | SHA-256 |"
echo "|---|---|"
grep -v -E '\.(md5|sha1|sha256|sha512)$' "$OUT/MANIFEST.sha256" | while read -r sum path; do
echo "| \`$(basename "$path")\` | \`$sum\` |"
done
echo
echo "</details>"
} | summary
128 changes: 128 additions & 0 deletions .github/scripts/release/build-artifacts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
#!/usr/bin/env bash
# Build everything that is published for one Spark profile, unsigned, and
# record digests. Runs the project's Maven build, so it runs only in a job
# that has no credentials.
#
# Environment:
# SRC checkout of the commit being released
# PROFILE spark-3.5 | spark-4.0 | spark-4.1
# BASE_VERSION the tag without its "v" (0.6.0)
# EXPECTED_VERSIONS the versions plan.sh announced, one per profile
# NATIVE_DIR the native job's artifact (build-native.sh output)
# OUT output directory; becomes the job's artifact:
# files/ the six files of this version, under the
# names they are published with
# build-info.txt profile, versions, tool versions
# SHA256SUMS digests of everything above
#
# The published version is <BASE_VERSION>_spark_<spark.version>, where
# spark.version is what the profile sets in pom.xml at the released commit,
# so the suffix always names the Spark version the jar was compiled against.
# It must be the version plan.sh worked out from the same pom.xml and showed
# in the Plan summary; if Maven disagrees, the build stops.
set -euo pipefail
here="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=lib.sh
. "$here/lib.sh"

: "${SRC:?}" "${PROFILE:?}" "${BASE_VERSION:?}" "${EXPECTED_VERSIONS:?}" "${NATIVE_DIR:?}" "${OUT:?}"

# Pinned here because they are invoked by coordinate, not through pom.xml.
VERSIONS_PLUGIN=org.codehaus.mojo:versions-maven-plugin:2.22.0
HELP_PLUGIN=org.apache.maven.plugins:maven-help-plugin:3.5.1

need mvn java unzip
spark_line "$PROFILE" > /dev/null
[[ "$BASE_VERSION" =~ $BASE_RE ]] || die "'$BASE_VERSION' is not a base version"
[ -f "$SRC/pom.xml" ] || die "no pom.xml in $SRC"

# --- the native jar built by this run, and nothing else -----------------------
verify_sums "$NATIVE_DIR"
t4j_version="$(pom_tantivy4java_version "$SRC/pom.xml")"
[ "$(kv "$NATIVE_DIR/native-inputs.txt" tantivy4java_version)" = "$t4j_version" ] \
|| die "the native artifact is not tantivy4java $t4j_version"
t4j_jar="tantivy4java-$t4j_version-$NATIVE_CLASSIFIER.jar"
[ -f "$NATIVE_DIR/repository/$t4j_jar" ] || die "the native artifact has no $t4j_jar"
t4j_sha="$(hash_of sha256 "$NATIVE_DIR/repository/$t4j_jar")"

dest="$M2_REPO/io/indextables/tantivy4java/$t4j_version"
[ ! -e "$dest" ] \
|| die "tantivy4java $t4j_version is already in the local Maven repository ($dest); the release build must take it from this run's native job only"
mkdir -p "$dest"
cp "$NATIVE_DIR/repository/"* "$dest/"

mvn_cmd=(mvn -B -ntp "-Dmaven.repo.local=$M2_REPO" "-Dplatform.classifier=$NATIVE_CLASSIFIER")

# --- version ------------------------------------------------------------------
spark_version="$(cd "$SRC" && "${mvn_cmd[@]}" -q "-P$PROFILE" "$HELP_PLUGIN:evaluate" \
-Dexpression=spark.version -DforceStdout | tail -n 1 | tr -d '[:space:]')"
version="${BASE_VERSION}_spark_${spark_version}"
check_version "$BASE_VERSION" "$PROFILE" "$version"
announced="$(profile_version "$EXPECTED_VERSIONS" "$PROFILE")"
[ "$version" = "$announced" ] \
|| die "Maven derives $version for $PROFILE, but the plan announced $announced"
echo "Building $ARTIFACT_ID $version (profile $PROFILE, Spark $spark_version)"

(cd "$SRC" && "${mvn_cmd[@]}" "$VERSIONS_PLUGIN:set" "-DnewVersion=$version" -DgenerateBackupPoms=false)

# --- build --------------------------------------------------------------------
# `verify` with the release profile produces the jar, the sources jar, the
# scaladoc jar, the shaded jar and the jar-with-dependencies. Signing is
# switched off (there is no key in this job) and nothing is deployed.
rm -f "$SRC/dependency-reduced-pom.xml"
(cd "$SRC" && "${mvn_cmd[@]}" "-P$PROFILE,release" -DskipTests -Dgpg.skip=true verify)

[ "$(hash_of sha256 "$dest/$t4j_jar")" = "$t4j_sha" ] \
|| die "$t4j_jar in the local Maven repository changed during the build"

# --- collect ------------------------------------------------------------------
# Target file -> published name. The published pom is the dependency-reduced
# pom that the shade plugin writes; the assembly is attached under the
# classifier "jar-with-dependencies" although its file in target/ carries the
# platform classifier as well. Both match what is on Maven Central for
# earlier releases.
p="$ARTIFACT_ID-$version"
t="$SRC/target"
rm -rf "$OUT"
mkdir -p "$OUT/files"
collect() { # collect <source file> <published name>
[ -f "$1" ] || die "the build did not produce $1"
cp "$1" "$OUT/files/$2"
}
collect "$SRC/dependency-reduced-pom.xml" "$p.pom"
collect "$t/$p.jar" "$p.jar"
collect "$t/$p-sources.jar" "$p-sources.jar"
collect "$t/$p-javadoc.jar" "$p-javadoc.jar"
collect "$t/$p-$NATIVE_CLASSIFIER-shaded.jar" "$p-$NATIVE_CLASSIFIER-shaded.jar"
collect "$t/$p-$NATIVE_CLASSIFIER-jar-with-dependencies.jar" "$p-jar-with-dependencies.jar"

# A jar this script does not know about means pom.xml attaches something new.
# Stop, so that publishing it (or not) is a decision and not an accident.
for f in "$t/$ARTIFACT_ID-"*.jar; do
case "$(basename "$f")" in
"$p.jar" | "$p-sources.jar" | "$p-javadoc.jar" | "$p-$NATIVE_CLASSIFIER-shaded.jar" | "$p-$NATIVE_CLASSIFIER-jar-with-dependencies.jar") ;;
*) die "the build produced $(basename "$f"), which the release scripts do not know; update ARTIFACT_SUFFIXES in lib.sh and build-artifacts.sh if it should be published" ;;
esac
done

{
echo "profile=$PROFILE"
echo "spark_version=$spark_version"
echo "version=$version"
echo "tantivy4java_version=$t4j_version"
echo "tantivy4java_jar_sha256=$t4j_sha"
echo "java=$("${JAVA_HOME:+$JAVA_HOME/bin/}java" -version 2>&1 | head -n 1)"
echo "maven=$(mvn --version 2> /dev/null | head -n 1)"
} > "$OUT/build-info.txt"
write_sums "$OUT"

output version "$version"
{
echo "### $ARTIFACT_ID $version ($PROFILE)"
echo
echo "| File | SHA-256 |"
echo "|---|---|"
grep ' files/' "$OUT/SHA256SUMS" | while read -r sum path; do
echo "| \`${path#files/}\` | \`$sum\` |"
done
} | summary
Loading
Loading