From 965ede004031ed5f4f2c802cafa61dca4d6d16cb Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Tue, 30 Jun 2026 19:45:35 +0200 Subject: [PATCH 1/8] build: stop bundling storm-autocreds in the binary distribution storm-autocreds pulls in the full Hadoop/HBase client dependency tree (~79 MB, 43 jars unique to it) but is only needed on secure (Kerberos) clusters and is off by default. Ship only the README, consistent with the other external/* connectors, and add bin/storm-autocreds-fetch to retrieve the plugin and its runtime dependencies from Maven Central into extlib-daemon on demand. Also removes the now-unused storm-autocreds-bin assembly module. --- bin/storm-autocreds-fetch | 134 ++++++++++++++++++ external/storm-autocreds/README.md | 101 +++++++++++++ .../src/main/assembly/binary.xml | 9 +- storm-dist/binary/pom.xml | 1 - storm-dist/binary/storm-autocreds-bin/pom.xml | 63 -------- .../src/main/assembly/storm-autocreds.xml | 33 ----- .../binary/storm-kafka-monitor-bin/pom.xml | 69 --------- .../src/main/assembly/storm-kafka-monitor.xml | 33 ----- 8 files changed, 241 insertions(+), 202 deletions(-) create mode 100755 bin/storm-autocreds-fetch create mode 100644 external/storm-autocreds/README.md delete mode 100644 storm-dist/binary/storm-autocreds-bin/pom.xml delete mode 100644 storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml delete mode 100644 storm-dist/binary/storm-kafka-monitor-bin/pom.xml delete mode 100644 storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml diff --git a/bin/storm-autocreds-fetch b/bin/storm-autocreds-fetch new file mode 100755 index 00000000000..d585be72b69 --- /dev/null +++ b/bin/storm-autocreds-fetch @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Fetch the storm-autocreds plugin and its (Hadoop/HBase) runtime dependencies +# into the daemon classpath, so Nimbus/Supervisor can populate and renew HDFS +# and HBase delegation tokens on a secure (Kerberos) cluster. +# +# These jars are intentionally NOT bundled in the binary distribution to keep it +# small; only secure-Hadoop deployments need them. See +# external/storm-autocreds/README.md for details. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: storm-autocreds-fetch [options] [-- ] + +Resolves org.apache.storm:storm-autocreds and its runtime dependencies from a +Maven repository (Maven Central by default) and copies them into the Storm +daemon classpath directory (extlib-daemon). + +Options: + --version Storm version to fetch (default: read from $STORM_HOME/RELEASE) + --dest Target directory (default: $STORM_HOME/extlib-daemon) + -h, --help Show this help + +Any arguments after "--" are passed through to Maven, e.g. to use an internal +mirror or an offline local repository: + storm-autocreds-fetch -- -s /path/settings.xml + storm-autocreds-fetch -- -Dmaven.repo.local=/path/to/offline-repo -o +EOF +} + +# Resolve symlinks so STORM_HOME is correct even when invoked via a link. +PRG="${0}" +while [ -h "${PRG}" ]; do + ls=$(ls -ld "${PRG}") + link=$(expr "${ls}" : '.*-> \(.*\)$') + if expr "${link}" : '/.*' > /dev/null; then + PRG="${link}" + else + PRG="$(dirname "${PRG}")/${link}" + fi +done +STORM_BIN_DIR=$(dirname "${PRG}") +STORM_HOME=$(cd "${STORM_BIN_DIR}/.." && pwd) + +VERSION="" +DEST="" +MVN_ARGS=() +while [ $# -gt 0 ]; do + case "${1}" in + --version) VERSION="${2}"; shift 2 ;; + --dest) DEST="${2}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + --) shift; MVN_ARGS=("$@"); break ;; + *) echo "Unknown option: ${1}" >&2; usage; exit 1 ;; + esac +done + +if [ -z "${VERSION}" ]; then + if [ -f "${STORM_HOME}/RELEASE" ]; then + VERSION=$(tr -d '[:space:]' < "${STORM_HOME}/RELEASE") + fi +fi +if [ -z "${VERSION}" ]; then + echo "Error: could not determine Storm version. Pass --version ." >&2 + exit 1 +fi + +if [ -z "${DEST}" ]; then + DEST="${STORM_HOME}/extlib-daemon" +fi + +MVN="${MAVEN_HOME:+${MAVEN_HOME}/bin/}mvn" +if ! command -v "${MVN}" > /dev/null 2>&1; then + echo "Error: '${MVN}' not found on PATH. Install Apache Maven or set MAVEN_HOME." >&2 + exit 1 +fi + +mkdir -p "${DEST}" + +# Use a throwaway POM that depends on storm-autocreds; copy-dependencies then +# pulls the exact runtime closure (honoring the exclusions declared in the +# published storm-autocreds POM). storm-client is 'provided' there and is +# correctly skipped, since it already ships in lib/. +TMP_DIR=$(mktemp -d) +trap 'rm -rf "${TMP_DIR}"' EXIT +cat > "${TMP_DIR}/pom.xml" < + 4.0.0 + org.apache.storm.tools + storm-autocreds-fetch + ${VERSION} + pom + + + org.apache.storm + storm-autocreds + ${VERSION} + + + +EOF + +echo "Fetching org.apache.storm:storm-autocreds:${VERSION} (runtime closure) into:" +echo " ${DEST}" +"${MVN}" -q -f "${TMP_DIR}/pom.xml" \ + org.apache.maven.plugins:maven-dependency-plugin:copy-dependencies \ + -DincludeScope=runtime \ + -DoutputDirectory="${DEST}" \ + ${MVN_ARGS[@]+"${MVN_ARGS[@]}"} + +echo "Done. ${DEST} now contains:" +ls -1 "${DEST}" | sed 's/^/ /' +echo +echo "Restart the Storm daemons (Nimbus, Supervisor) to pick up the new classpath," +echo "then configure the autocreds plugins in storm.yaml. See" +echo "external/storm-autocreds/README.md for the required settings." diff --git a/external/storm-autocreds/README.md b/external/storm-autocreds/README.md new file mode 100644 index 00000000000..02459352994 --- /dev/null +++ b/external/storm-autocreds/README.md @@ -0,0 +1,101 @@ +# Storm Auto Credentials (HDFS / HBase) + +`storm-autocreds` lets Storm automatically acquire, distribute and renew +**Hadoop delegation tokens** so that topologies can talk to a secure (Kerberos) +HDFS or HBase cluster without distributing keytabs to every worker host. + +* On topology submission, **Nimbus** obtains delegation tokens on behalf of the + submitting user and ships them with the topology. +* **Workers** unpack the tokens into their `Subject` / `UserGroupInformation`. +* **Nimbus** periodically renews the tokens for long-running topologies. + +See `docs/SECURITY.md` ("Automatic Credentials Push and Renewal") for the full +design. + +## Why the jars are not bundled + +Because these plugins run on the **daemon** classpath (Nimbus/Supervisor) and +pull in the full Hadoop and HBase client dependency trees, they are **not** +shipped inside the binary distribution — only secure-Hadoop deployments need +them, and bundling them would bloat the distribution for everyone. This is the +same convention used by the other `external/*` connectors. + +## Installing + +The plugins must be present on the **daemon** classpath, i.e. in +`$STORM_HOME/extlib-daemon` on Nimbus and the Supervisors. + +### Option 1 — use the helper script (recommended) + +The distribution ships a helper that resolves `storm-autocreds` and its runtime +dependencies from Maven Central and copies them into `extlib-daemon`: + +```bash +$STORM_HOME/bin/storm-autocreds-fetch +``` + +It detects the Storm version from `$STORM_HOME/RELEASE`. Useful options: + +```bash +# explicit version / target directory +bin/storm-autocreds-fetch --version 3.0.0 --dest /opt/storm/extlib-daemon + +# pass extra arguments through to Maven (internal mirror / offline repo) +bin/storm-autocreds-fetch -- -s /etc/maven/settings.xml +bin/storm-autocreds-fetch -- -Dmaven.repo.local=/srv/offline-repo -o +``` + +Maven must be available on the host running the script (it does not have to be +installed on the cluster nodes — you can run it once and copy the resulting jars +to every daemon host). + +### Option 2 — build from source + +```bash +mvn -pl external/storm-autocreds -am package +cp external/storm-autocreds/target/storm-autocreds-*.jar \ + $(find ~/.m2 -name 'hadoop-auth-*.jar' -o -name 'hbase-client-*.jar') \ + $STORM_HOME/extlib-daemon/ +``` + +(Prefer Option 1 — it resolves the complete, correct dependency closure for you.) + +Restart Nimbus and the Supervisors after adding the jars so the new classpath +takes effect. + +## Configuring + +Add the following to `storm.yaml`. The `*Nimbus` classes run on Nimbus (acquire +and renew tokens); the non-`Nimbus` classes run in the worker (unpack tokens). + +```yaml +# Worker side: unpack the tokens into the worker Subject. +topology.auto-credentials: + - org.apache.storm.hdfs.security.AutoHDFS + - org.apache.storm.hbase.security.AutoHBase + +# Nimbus side: obtain the tokens on behalf of the submitter. +nimbus.autocredential.plugins.classes: + - org.apache.storm.hdfs.security.AutoHDFSNimbus + - org.apache.storm.hbase.security.AutoHBaseNimbus + +# Nimbus side: renew the tokens for long-running topologies. +nimbus.credential.renewers.classes: + - org.apache.storm.hdfs.security.AutoHDFSNimbus + - org.apache.storm.hbase.security.AutoHBaseNimbus +``` + +Relevant credential settings: + +| Setting | Purpose | +|---|---| +| `hdfs.keytab.file` / `hdfs.kerberos.principal` | Nimbus principal used to fetch HDFS tokens | +| `hdfs.kerberos.principal` | HDFS service principal | +| `hbase.keytab.file` / `hbase.kerberos.principal` | Nimbus principal used to fetch HBase tokens | +| `topology.hdfs.uri` | NameNode URI (defaults to the cluster `fs.defaultFS`) | +| `hdfsCredentialsConfigKeys` / `hbaseCredentialsConfigKeys` | optional list of per-cluster config keys when talking to multiple clusters | + +Use only the HDFS or only the HBase entries if you need just one of them. + +For the full secure-cluster setup (Kerberos, impersonation, ACLs) see +`docs/SECURITY.md`. diff --git a/storm-dist/binary/final-package/src/main/assembly/binary.xml b/storm-dist/binary/final-package/src/main/assembly/binary.xml index ee5cb60e3ed..2f4afedc228 100644 --- a/storm-dist/binary/final-package/src/main/assembly/binary.xml +++ b/storm-dist/binary/final-package/src/main/assembly/binary.xml @@ -241,12 +241,15 @@ - + - ${project.basedir}/../storm-autocreds-bin/target/autocreds/autocreds/lib-autocreds + ${project.basedir}/../../../external/storm-autocreds external/storm-autocreds - *jar + README.* diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml index 94209592e3c..166ee9442d2 100644 --- a/storm-dist/binary/pom.xml +++ b/storm-dist/binary/pom.xml @@ -47,7 +47,6 @@ storm-client-bin storm-webapp-bin - storm-autocreds-bin storm-submit-tools-bin storm-kafka-monitor-bin diff --git a/storm-dist/binary/storm-autocreds-bin/pom.xml b/storm-dist/binary/storm-autocreds-bin/pom.xml deleted file mode 100644 index 8b42b902fff..00000000000 --- a/storm-dist/binary/storm-autocreds-bin/pom.xml +++ /dev/null @@ -1,63 +0,0 @@ - - - - 4.0.0 - - org.apache.storm - apache-storm-bin - 3.0.0-SNAPSHOT - - storm-autocreds-bin - pom - - - Storm Autocreds Binary - - - org.apache.storm - storm-autocreds - ${project.version} - - - - - autocreds - - - org.apache.maven.plugins - maven-assembly-plugin - - - prepare-package - - single - - - - - false - false - - ${project.basedir}/src/main/assembly/storm-autocreds.xml - - false - - - - - diff --git a/storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml b/storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml deleted file mode 100644 index 1cd914cb7b7..00000000000 --- a/storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - storm-autocreds-bin - - dir - - - - false - lib-autocreds - false - - - diff --git a/storm-dist/binary/storm-kafka-monitor-bin/pom.xml b/storm-dist/binary/storm-kafka-monitor-bin/pom.xml deleted file mode 100644 index 4d9e569a95a..00000000000 --- a/storm-dist/binary/storm-kafka-monitor-bin/pom.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - 4.0.0 - - org.apache.storm - apache-storm-bin - 3.0.0-SNAPSHOT - - storm-kafka-monitor-bin - pom - - - Storm Kafka Monitor Binary - - - org.apache.storm - storm-kafka-monitor - ${project.version} - - - - org.apache.kafka - kafka-clients - compile - - - - - kafka-monitor - - - org.apache.maven.plugins - maven-assembly-plugin - - - prepare-package - - single - - - - - false - false - - ${project.basedir}/src/main/assembly/storm-kafka-monitor.xml - - false - - - - - diff --git a/storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml b/storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml deleted file mode 100644 index 021b472c87f..00000000000 --- a/storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - storm-kafka-monitor-bin - - dir - - - - false - lib-kafka-monitor - false - - - From 1b3279af7be672710d14958e8f66ad6776daa24c Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Tue, 30 Jun 2026 19:58:51 +0200 Subject: [PATCH 2/8] build: stop bundling storm-kafka-monitor in the binary distribution The storm-kafka-monitor jars (and their Kafka client dependencies, ~38 MB) are only needed to display Kafka spout lag in the UI or to run the bin/storm-kafka-monitor command. Ship only the README, consistent with the other external/* connectors, and add bin/storm-kafka-monitor-fetch to retrieve the tool and its runtime dependencies from Maven Central into lib-tools/storm-kafka-monitor on demand. Guard the UI against the jars being absent: TopologySpoutLag now detects whether storm-kafka-monitor is installed and, when it is not, surfaces an actionable message (and logs it once) instead of failing the lag shell-out. The bin/storm-kafka-monitor wrapper prints the same hint instead of a ClassNotFound error. Also removes the now-unused storm-kafka-monitor-bin assembly module. --- bin/storm-kafka-monitor | 9 +- bin/storm-kafka-monitor-fetch | 133 ++++++++++++++++++ external/storm-kafka-monitor/README.md | 24 ++++ .../apache/storm/utils/TopologySpoutLag.java | 36 ++++- .../src/main/assembly/binary.xml | 11 +- storm-dist/binary/pom.xml | 1 - 6 files changed, 207 insertions(+), 7 deletions(-) create mode 100755 bin/storm-kafka-monitor-fetch diff --git a/bin/storm-kafka-monitor b/bin/storm-kafka-monitor index 9bd11054cb5..a586c1d8d58 100755 --- a/bin/storm-kafka-monitor +++ b/bin/storm-kafka-monitor @@ -49,4 +49,11 @@ if [ -z "$JAVA_HOME" ]; then else JAVA="$JAVA_HOME/bin/java" fi -exec $JAVA $STORM_JAAS_CONF_PARAM $STORM_JAR_JVM_OPTS -cp "$STORM_BASE_DIR/lib-tools/storm-kafka-monitor/*" org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@" +# The storm-kafka-monitor jars are not bundled in the distribution; they are fetched on demand. +KAFKA_MONITOR_LIB="$STORM_BASE_DIR/lib-tools/storm-kafka-monitor" +if ! ls "$KAFKA_MONITOR_LIB"/*.jar >/dev/null 2>&1; then + echo "storm-kafka-monitor is not installed (no jars in $KAFKA_MONITOR_LIB)." >&2 + echo "Run '$STORM_BIN_DIR/storm-kafka-monitor-fetch' to download it, then retry." >&2 + exit 1 +fi +exec $JAVA $STORM_JAAS_CONF_PARAM $STORM_JAR_JVM_OPTS -cp "$KAFKA_MONITOR_LIB/*" org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@" diff --git a/bin/storm-kafka-monitor-fetch b/bin/storm-kafka-monitor-fetch new file mode 100755 index 00000000000..573312d87ef --- /dev/null +++ b/bin/storm-kafka-monitor-fetch @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Fetch the storm-kafka-monitor tool and its (Kafka client) runtime dependencies +# into lib-tools/storm-kafka-monitor, enabling the "Kafka spout lag" display in +# the Storm UI and the bin/storm-kafka-monitor command. +# +# These jars are intentionally NOT bundled in the binary distribution to keep it +# small; they are only needed when running Kafka spouts and wanting lag info. The +# UI degrades gracefully when they are absent. See +# external/storm-kafka-monitor/README.md for details. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: storm-kafka-monitor-fetch [options] [-- ] + +Resolves org.apache.storm:storm-kafka-monitor and its runtime dependencies from +a Maven repository (Maven Central by default) and copies them into +lib-tools/storm-kafka-monitor. + +Options: + --version Storm version to fetch (default: read from $STORM_HOME/RELEASE) + --dest Target directory (default: $STORM_HOME/lib-tools/storm-kafka-monitor) + -h, --help Show this help + +Any arguments after "--" are passed through to Maven, e.g. to use an internal +mirror or an offline local repository: + storm-kafka-monitor-fetch -- -s /path/settings.xml + storm-kafka-monitor-fetch -- -Dmaven.repo.local=/path/to/offline-repo -o +EOF +} + +# Resolve symlinks so STORM_HOME is correct even when invoked via a link. +PRG="${0}" +while [ -h "${PRG}" ]; do + ls=$(ls -ld "${PRG}") + link=$(expr "${ls}" : '.*-> \(.*\)$') + if expr "${link}" : '/.*' > /dev/null; then + PRG="${link}" + else + PRG="$(dirname "${PRG}")/${link}" + fi +done +STORM_BIN_DIR=$(dirname "${PRG}") +STORM_HOME=$(cd "${STORM_BIN_DIR}/.." && pwd) + +VERSION="" +DEST="" +MVN_ARGS=() +while [ $# -gt 0 ]; do + case "${1}" in + --version) VERSION="${2}"; shift 2 ;; + --dest) DEST="${2}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + --) shift; MVN_ARGS=("$@"); break ;; + *) echo "Unknown option: ${1}" >&2; usage; exit 1 ;; + esac +done + +if [ -z "${VERSION}" ]; then + if [ -f "${STORM_HOME}/RELEASE" ]; then + VERSION=$(tr -d '[:space:]' < "${STORM_HOME}/RELEASE") + fi +fi +if [ -z "${VERSION}" ]; then + echo "Error: could not determine Storm version. Pass --version ." >&2 + exit 1 +fi + +if [ -z "${DEST}" ]; then + DEST="${STORM_HOME}/lib-tools/storm-kafka-monitor" +fi + +MVN="${MAVEN_HOME:+${MAVEN_HOME}/bin/}mvn" +if ! command -v "${MVN}" > /dev/null 2>&1; then + echo "Error: '${MVN}' not found on PATH. Install Apache Maven or set MAVEN_HOME." >&2 + exit 1 +fi + +mkdir -p "${DEST}" + +# Use a throwaway POM that depends on storm-kafka-monitor; copy-dependencies then +# pulls the exact runtime closure. The artifact itself is a direct dependency and +# is therefore copied too. +TMP_DIR=$(mktemp -d) +trap 'rm -rf "${TMP_DIR}"' EXIT +cat > "${TMP_DIR}/pom.xml" < + 4.0.0 + org.apache.storm.tools + storm-kafka-monitor-fetch + ${VERSION} + pom + + + org.apache.storm + storm-kafka-monitor + ${VERSION} + + + +EOF + +echo "Fetching org.apache.storm:storm-kafka-monitor:${VERSION} (runtime closure) into:" +echo " ${DEST}" +"${MVN}" -q -f "${TMP_DIR}/pom.xml" \ + org.apache.maven.plugins:maven-dependency-plugin:copy-dependencies \ + -DincludeScope=runtime \ + -DoutputDirectory="${DEST}" \ + ${MVN_ARGS[@]+"${MVN_ARGS[@]}"} + +echo "Done. ${DEST} now contains:" +ls -1 "${DEST}" | sed 's/^/ /' +echo +echo "Restart the Storm UI to enable Kafka spout lag display, or run" +echo "bin/storm-kafka-monitor directly. See external/storm-kafka-monitor/README.md." diff --git a/external/storm-kafka-monitor/README.md b/external/storm-kafka-monitor/README.md index a483f4bdef9..8e5bd37ef4e 100644 --- a/external/storm-kafka-monitor/README.md +++ b/external/storm-kafka-monitor/README.md @@ -2,6 +2,30 @@ Tool to query kafka spout lags and show in Storm UI +## Installation + +The storm-kafka-monitor jars (and their Kafka client dependencies) are **not** +bundled in the binary distribution to keep it small — they are only needed to +display Kafka spout lag in the UI or to run the `storm-kafka-monitor` command. +The Storm UI degrades gracefully when they are absent (no lag is shown and a +hint is logged once). + +To enable it, install the jars on the UI host with the helper script, then +restart the UI: + +```bash +$STORM_HOME/bin/storm-kafka-monitor-fetch +``` + +It resolves `org.apache.storm:storm-kafka-monitor` and its runtime dependencies +from Maven Central into `lib-tools/storm-kafka-monitor`. Useful options: + +```bash +bin/storm-kafka-monitor-fetch --version 3.0.0 +# pass extra arguments through to Maven (internal mirror / offline repo) +bin/storm-kafka-monitor-fetch -- -s /etc/maven/settings.xml +``` + ## Usage This tool provides a way to query kafka offsets that the spout has consumed successfully and the latest offsets in kafka. It provides an easy way to see how the topology is performing. It is a command line diff --git a/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java index 7d8d7bbc8d0..a230e1e7a25 100644 --- a/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java +++ b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java @@ -54,6 +54,11 @@ public class TopologySpoutLag { BOOTSTRAP_CONFIG, SECURITY_PROTOCOL_CONFIG)); private static final Logger LOGGER = LoggerFactory.getLogger(TopologySpoutLag.class); + // The storm-kafka-monitor jars are not bundled in the binary distribution; operators install them + // on demand (bin/storm-kafka-monitor-fetch). Log the "not installed" hint at most once to avoid + // spamming the UI logs, which poll the lag endpoint periodically. + private static volatile boolean warnedMonitorMissing = false; + public static Map> lag(StormTopology stormTopology, Map topologyConf) { Map> result = new HashMap<>(); Map spouts = stormTopology.get_spouts(); @@ -69,6 +74,24 @@ public static Map> lag(StormTopology stormTopology, return result; } + /** + * Checks whether the storm-kafka-monitor jars (invoked by bin/storm-kafka-monitor) are present. + * They are not bundled in the binary distribution and are fetched on demand, so the UI must + * degrade gracefully when they are absent rather than failing the lag shell-out. + * + * @return true if the monitor appears installed, or if STORM_BASE_DIR is unknown (in which case + * the legacy behavior of attempting the shell-out is preserved). + */ + private static boolean isKafkaMonitorInstalled() { + String stormHomeDir = System.getenv("STORM_BASE_DIR"); + if (stormHomeDir == null) { + return true; + } + File libDir = new File(new File(stormHomeDir, "lib-tools"), "storm-kafka-monitor"); + File[] jars = libDir.listFiles((dir, name) -> name.endsWith(".jar")); + return jars != null && jars.length > 0; + } + private static List getCommandLineOptionsForNewKafkaSpout(Map jsonConf) { LOGGER.debug("json configuration: {}", jsonConf); @@ -166,7 +189,18 @@ private static Map getLagResultForKafka(String spoutId, SpoutSpe LOGGER.debug("Command to run: {}", commands); // if commands contains one or more null value, spout is compiled with lower version of storm-kafka-client - if (!commands.contains(null)) { + if (!commands.contains(null) && !isKafkaMonitorInstalled()) { + errorMsg = "Kafka spout lag monitoring is unavailable because the storm-kafka-monitor " + + "jars are not installed. They are no longer bundled in the binary distribution; " + + "run 'bin/storm-kafka-monitor-fetch' on the UI host (and restart the UI) to enable it."; + if (!warnedMonitorMissing) { + warnedMonitorMissing = true; + LOGGER.info(errorMsg); + } + if (extraPropertiesFile != null) { + extraPropertiesFile.delete(); + } + } else if (!commands.contains(null)) { try { String resultFromMonitor = new ShellCommandRunnerImpl().execCommand(commands.toArray(new String[0])); diff --git a/storm-dist/binary/final-package/src/main/assembly/binary.xml b/storm-dist/binary/final-package/src/main/assembly/binary.xml index 2f4afedc228..e41226110cf 100644 --- a/storm-dist/binary/final-package/src/main/assembly/binary.xml +++ b/storm-dist/binary/final-package/src/main/assembly/binary.xml @@ -232,12 +232,15 @@ - + - ${project.basedir}/../storm-kafka-monitor-bin/target/kafka-monitor/kafka-monitor/lib-kafka-monitor - lib-tools/storm-kafka-monitor + ${project.basedir}/../../../external/storm-kafka-monitor + external/storm-kafka-monitor - *jar + README.* diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml index 166ee9442d2..3b58038d172 100644 --- a/storm-dist/binary/pom.xml +++ b/storm-dist/binary/pom.xml @@ -48,7 +48,6 @@ storm-client-bin storm-webapp-bin storm-submit-tools-bin - storm-kafka-monitor-bin final-package From f054636cfdab754727bb1ec56508eabc60b13946 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Tue, 30 Jun 2026 20:05:14 +0200 Subject: [PATCH 3/8] build: add lib-common to the daemon and worker classpaths Prepares de-duplication of the jars shared by the daemon (lib) and worker (lib-worker) classpaths into a single lib-common directory. storm.py now includes lib-common on both classpaths; when the directory is absent (older layouts) it contributes nothing, so the change is backward compatible. --- bin/storm.py | 11 ++++++++--- bin/test_storm.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/bin/storm.py b/bin/storm.py index 81d6e4e4d8e..eb1afa2f85f 100755 --- a/bin/storm.py +++ b/bin/storm.py @@ -107,6 +107,7 @@ def confvalue(name, storm_config_opts, extrapaths, overriding_conf_file=None, da def get_classpath(extrajars, daemon=True, client=False): ret = get_wildcard_dir(STORM_DIR) + ret.extend(get_wildcard_dir(STORM_COMMON_LIB_DIR)) if client: ret.extend(get_wildcard_dir(STORM_WORKER_LIB_DIR)) else: @@ -125,9 +126,9 @@ def get_classpath(extrajars, daemon=True, client=False): def init_storm_env(within_unittest=False): global NORMAL_CLASS_PATH, STORM_DIR, USER_CONF_DIR, STORM_CONF_DIR, STORM_WORKER_LIB_DIR, STORM_LIB_DIR,\ - STORM_TOOLS_LIB_DIR, STORM_WEBAPP_LIB_DIR, STORM_BIN_DIR, STORM_LOG4J2_CONF_DIR, STORM_SUPERVISOR_LOG_FILE,\ - CLUSTER_CONF_DIR, JAR_JVM_OPTS, JAVA_HOME, JAVA_CMD, CONF_FILE, STORM_EXT_CLASSPATH, \ - STORM_EXT_CLASSPATH_DAEMON, LOCAL_TTL_DEFAULT + STORM_COMMON_LIB_DIR, STORM_TOOLS_LIB_DIR, STORM_WEBAPP_LIB_DIR, STORM_BIN_DIR, STORM_LOG4J2_CONF_DIR,\ + STORM_SUPERVISOR_LOG_FILE, CLUSTER_CONF_DIR, JAR_JVM_OPTS, JAVA_HOME, JAVA_CMD, CONF_FILE, \ + STORM_EXT_CLASSPATH, STORM_EXT_CLASSPATH_DAEMON, LOCAL_TTL_DEFAULT NORMAL_CLASS_PATH = cygpath if sys.platform == 'cygwin' else identity STORM_DIR = os.sep.join(os.path.realpath( __file__ ).split(os.sep)[:-2]) @@ -141,6 +142,10 @@ def init_storm_env(within_unittest=False): STORM_WORKER_LIB_DIR = os.path.join(STORM_DIR, "lib-worker") STORM_LIB_DIR = os.path.join(STORM_DIR, "lib") + # Jars shared by the daemon (lib) and worker (lib-worker) classpaths are de-duplicated into + # lib-common to keep the distribution small. It is added to both classpaths; absent in older + # layouts, in which case it contributes nothing. + STORM_COMMON_LIB_DIR = os.path.join(STORM_DIR, "lib-common") STORM_TOOLS_LIB_DIR = os.path.join(STORM_DIR, "lib-tools") STORM_WEBAPP_LIB_DIR = os.path.join(STORM_DIR, "lib-webapp") diff --git a/bin/test_storm.py b/bin/test_storm.py index 11c8057b0a8..a4466991909 100644 --- a/bin/test_storm.py +++ b/bin/test_storm.py @@ -66,6 +66,19 @@ def test_get_classpath(self): expected = ":".join(extrajars) self.assertEqual(s[-len(expected):], expected) + def test_get_classpath_includes_lib_common(self): + extrajars = [] + # When lib-common exists, it is included on both the daemon and the client classpaths. + storm.STORM_COMMON_LIB_DIR = storm.STORM_BIN_DIR + expected = os.path.join(storm.STORM_BIN_DIR, "*") + for client in (True, False): + cp = storm.get_classpath(extrajars, daemon=True, client=client) + self.assertIn(expected, cp.split(os.pathsep)) + # When it does not exist, it contributes nothing (backward compatible with older layouts). + storm.STORM_COMMON_LIB_DIR = os.path.join(storm.STORM_DIR, "no-such-lib-common") + cp = storm.get_classpath(extrajars, daemon=True, client=False) + self.assertNotIn(os.path.join(storm.STORM_COMMON_LIB_DIR, "*"), cp.split(os.pathsep)) + def test_resolve_dependencies(self): artifacts = "org.apache.commons.commons-api" artifact_repositories = "maven-central" From d33bdb11921bb05e8ab51fa3a05e4948b423823d Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Tue, 30 Jun 2026 20:10:36 +0200 Subject: [PATCH 4/8] build: add dedup-libs script to share daemon/worker jars via lib-common The worker classpath (lib-worker) is a byte-identical subset of the daemon classpath (lib). dedup-libs.py moves the shared jars into a single lib-common directory and removes the duplicate copies from lib, reclaiming ~71 MB. It only de-duplicates byte-identical jars (same name and sha-256), so a version mismatch is never silently merged; tool classpaths (lib-tools/*, lib-webapp) are left untouched. Paired with the lib-common classpath support in bin/storm.py. Wiring this into the binary assembly is a follow-up (it must be validated by a full -Pdist distribution build). --- .../src/main/scripts/dedup-libs.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100755 storm-dist/binary/final-package/src/main/scripts/dedup-libs.py diff --git a/storm-dist/binary/final-package/src/main/scripts/dedup-libs.py b/storm-dist/binary/final-package/src/main/scripts/dedup-libs.py new file mode 100755 index 00000000000..d6af3ea107d --- /dev/null +++ b/storm-dist/binary/final-package/src/main/scripts/dedup-libs.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# De-duplicates the jars shared by the daemon (lib) and worker (lib-worker) +# classpaths of an assembled Storm distribution into a single lib-common +# directory, to keep the distribution small. +# +# The worker classpath is a strict subset of the daemon classpath, so every +# lib-worker jar is moved into lib-common; any byte-identical copy in lib is +# then removed. bin/storm.py adds lib-common to BOTH classpaths, so: +# daemon classpath = lib-common + lib (unchanged jar set) +# worker classpath = lib-common (+ lib-worker, now empty) +# +# Only byte-identical jars (same name AND same sha-256) are de-duplicated, so a +# version mismatch is never silently merged. Tool classpaths (lib-tools/*, +# lib-webapp) are intentionally left untouched: their wrappers do not include +# lib-common, so their jars must stay in place. + +import hashlib +import os +import shutil +import sys + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def jars(directory): + if not os.path.isdir(directory): + return {} + return {f: os.path.join(directory, f) for f in os.listdir(directory) if f.endswith(".jar")} + + +def dedup(dist_root): + lib = os.path.join(dist_root, "lib") + worker = os.path.join(dist_root, "lib-worker") + common = os.path.join(dist_root, "lib-common") + + # Absorb the worker jars into lib-common. This handles two layouts: + # - exploded distribution: lib-worker is present, lib-common is created here; + # - staged build: lib-common is already populated, lib-worker is absent. + worker_jars = jars(worker) + if worker_jars: + os.makedirs(common, exist_ok=True) + for name, src in sorted(worker_jars.items()): + dst = os.path.join(common, name) + if not os.path.exists(dst): + shutil.move(src, dst) + else: + os.remove(src) + # lib-worker is now empty; remove it so the layout is unambiguous. + if os.path.isdir(worker) and not os.listdir(worker): + os.rmdir(worker) + + common_jars = jars(common) + if not common_jars: + print(f"dedup-libs: no jars in {common} (and none in {worker}); nothing to do") + return 0 + + lib_jars = jars(lib) + reclaimed = 0 + removed = 0 + for name, common_copy in sorted(common_jars.items()): + # Drop the byte-identical copy from the daemon lib dir, if present. + lib_copy = lib_jars.get(name) + if lib_copy and os.path.exists(lib_copy) and sha256(lib_copy) == sha256(common_copy): + reclaimed += os.path.getsize(lib_copy) + os.remove(lib_copy) + removed += 1 + + print(f"dedup-libs: lib-common has {len(common_jars)} jar(s); " + f"removed {removed} duplicate(s) from lib, reclaimed {reclaimed / (1024 * 1024):.1f} MB") + return 0 + + +def main(argv): + if len(argv) != 2: + print("Usage: dedup-libs.py ", file=sys.stderr) + return 2 + return dedup(argv[1]) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From 21707008f553c042a6574ab1759d26bf9b99f822 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Tue, 30 Jun 2026 20:13:57 +0200 Subject: [PATCH 5/8] build: wire lib-common de-duplication into the binary assembly final-package now stages the daemon jars (copy-dependencies -> staging/lib) and the shared jars (storm-client-bin's lib-worker -> staging/lib-common), runs dedup-libs.py to remove the byte-identical copies from lib, and the assembly packages the staged lib/ and lib-common/ directories. The storm-client-bin tree (which only carried lib-worker) is no longer copied directly. Verified through the prepare-package phase: staging/lib = 48 jars, staging/ lib-common = 40 jars, zero overlap, full 88-jar daemon set preserved across lib + lib-common, ~71 MB reclaimed. The final tar/zip packaging requires a full -Pdist (native) distribution build and must be validated in CI / on Linux. --- storm-dist/binary/final-package/pom.xml | 67 +++++++++++++++++++ .../src/main/assembly/binary.xml | 26 +++---- 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/storm-dist/binary/final-package/pom.xml b/storm-dist/binary/final-package/pom.xml index 8471b307c62..dce19eaf6ed 100644 --- a/storm-dist/binary/final-package/pom.xml +++ b/storm-dist/binary/final-package/pom.xml @@ -43,6 +43,73 @@ apache-storm-${project.version} + + + org.apache.maven.plugins + maven-dependency-plugin + + + stage-daemon-lib + prepare-package + + copy-dependencies + + + runtime + ${project.build.directory}/staging/lib + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + stage-common-lib + prepare-package + + copy-resources + + + ${project.build.directory}/staging/lib-common + + + ${project.basedir}/../storm-client-bin/target/client/client/lib-worker + + *.jar + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + dedup-libs + prepare-package + + exec + + + python3 + + ${project.basedir}/src/main/scripts/dedup-libs.py + ${project.build.directory}/staging + + + + + org.apache.maven.plugins maven-assembly-plugin diff --git a/storm-dist/binary/final-package/src/main/assembly/binary.xml b/storm-dist/binary/final-package/src/main/assembly/binary.xml index e41226110cf..5e62e2bc00d 100644 --- a/storm-dist/binary/final-package/src/main/assembly/binary.xml +++ b/storm-dist/binary/final-package/src/main/assembly/binary.xml @@ -24,21 +24,23 @@ zip - - - - false - lib - false - - - + - ${project.basedir}/../storm-client-bin/target/client/client/ - . + ${project.build.directory}/staging/lib + lib - */** + *.jar + + + + ${project.build.directory}/staging/lib-common + lib-common + + *.jar From 15a92351032beadab4823d0bb013ac248200de65 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Wed, 1 Jul 2026 08:41:45 +0200 Subject: [PATCH 6/8] test: add lib-common to expected classpaths in storm CLI tests The binary distribution now de-duplicates jars shared by the daemon and worker classpaths into a lib-common directory, which bin/storm.py adds to the classpath after the storm home wildcard. Update the expected classpath assertions in test_storm_cli.py to include lib-common. --- storm-client/test/py/test_storm_cli.py | 50 +++++++++++++------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/storm-client/test/py/test_storm_cli.py b/storm-client/test/py/test_storm_cli.py index 2815de42f1c..8322897b04e 100644 --- a/storm-client/test/py/test_storm_cli.py +++ b/storm-client/test/py/test_storm_cli.py @@ -62,7 +62,7 @@ def test_jar_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=+topology.blobstore.map%3D%27%7B%22key1%22%3A%7B%22localname%22%3A%22blob_file%22%2C+%22uncompress%22%3Afalse%7D%2C%22key2%22%3A%7B%7D%7D%27', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib-worker:' + self.storm_dir + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib-worker:' + self.storm_dir + '/extlib:example/storm-starter/storm-starter-topologies-*.jar:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin:./external/storm-redis/storm-redis-1.1.0.jar:./external/storm-kafka-client/storm-kafka-client-1.1.0.jar"', '-Dstorm.jar=example/storm-starter/storm-starter-topologies-*.jar', '-Dstorm.dependency.jars=./external/storm-redis/storm-redis-1.1.0.jar,./external/storm-kafka-client/storm-kafka-client-1.1.0.jar"', '-Dstorm.dependency.artifacts={}', 'org.apache.storm.starter.RollingTopWords', 'blobstore-remote2', 'remote' @@ -80,7 +80,7 @@ def test_jar_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib-worker:' + self.storm_dir + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib-worker:' + self.storm_dir + '/extlib:/path/to/jar.jar:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin:', '-Dstorm.jar=/path/to/jar.jar', '-Dstorm.dependency.jars=', '-Dstorm.dependency.artifacts={}', 'some.Topology.Class', '-name', 'run-topology', 'randomArgument', '-randomFlag', 'randomFlagValue', @@ -93,7 +93,7 @@ def test_localconfvalue_command(self): self.base_test( ["storm", "localconfvalue", "conf_name"], self.mock_popen, mock.call([ self.java_cmd, '-client', '-Dstorm.options=', - '-Dstorm.conf.file=', '-cp', self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir +'/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf', + '-Dstorm.conf.file=', '-cp', self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir +'/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf', 'org.apache.storm.command.ConfigValue', 'conf_name' ], stdout=-1 ) @@ -103,7 +103,7 @@ def test_remoteconfvalue_command(self): self.base_test( ["storm", "remoteconfvalue", "conf_name"], self.mock_popen, mock.call([ self.java_cmd, '-client', '-Dstorm.options=', - '-Dstorm.conf.file=', '-cp', self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf', + '-Dstorm.conf.file=', '-cp', self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf', 'org.apache.storm.command.ConfigValue', 'conf_name' ], stdout=-1 ) @@ -124,7 +124,7 @@ def test_local_command(self): self.java_cmd, '-client','-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:example/storm-starter/storm-starter-topologies-*.jar:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin:./external/storm-redis/storm-redis-1.1.0.jar:./external/storm-kafka-client/storm-kafka-client-1.1.0.jar"', @@ -144,7 +144,7 @@ def test_kill_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.KillTopology', 'doomed_topology' ]) ) @@ -157,7 +157,7 @@ def test_upload_credentials_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=test%3Dtest', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=/some/other/storm.yaml', - '-cp', self.storm_dir + '/*:' + self.storm_dir + '/lib:' + + '-cp', self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + @@ -173,7 +173,7 @@ def test_blobstore_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.Blobstore', 'create', 'mytopo:data.tgz', '-f', 'data.tgz', '-a', 'u:alice:rwa,u:bob:rw,o::r']) @@ -188,7 +188,7 @@ def test_blobstore_command(self): '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.Blobstore', 'list']) ) @@ -202,7 +202,7 @@ def test_blobstore_command(self): '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.Blobstore', 'list', 'wordstotrack']) ) @@ -215,7 +215,7 @@ def test_blobstore_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.Blobstore', 'update', '-f', '/wordsToTrack.list', 'wordstotrack']) @@ -229,7 +229,7 @@ def test_blobstore_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.Blobstore', 'cat', 'wordstotrack']) ) @@ -242,7 +242,7 @@ def test_activate_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.Activate', 'doomed_topology' ]) @@ -256,7 +256,7 @@ def test_deactivate_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.Deactivate', 'doomed_topology' ]) @@ -270,7 +270,7 @@ def test_rebalance_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.Rebalance', 'doomed_topology' ]) @@ -284,7 +284,7 @@ def test_list_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.ListTopologies' ]) @@ -298,7 +298,7 @@ def test_nimbus_command(self): self.java_cmd, '-server', '-Ddaemon.name=nimbus', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf', '-Djava.deserialization.disabled=true', '-Dlogfile.name=nimbus.log', '-Dlog4j.configurationFile=' + self.storm_dir + '/log4j2/cluster.xml', @@ -314,7 +314,7 @@ def test_supervisor_command(self): self.java_cmd, '-server', '-Ddaemon.name=supervisor', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf', '-Djava.deserialization.disabled=true', '-Dlogfile.name=supervisor.log', '-Dlog4j.configurationFile=' + self.storm_dir + '/log4j2/cluster.xml', @@ -330,7 +330,7 @@ def test_pacemaker_command(self): self.java_cmd, '-server', '-Ddaemon.name=pacemaker', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf', '-Djava.deserialization.disabled=true', '-Dlogfile.name=pacemaker.log', '-Dlog4j.configurationFile=' + self.storm_dir + '/log4j2/cluster.xml', @@ -346,7 +346,7 @@ def test_ui_command(self): self.java_cmd, '-server', '-Ddaemon.name=ui', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/lib-webapp:' + self.storm_dir + '/conf', '-Djava.deserialization.disabled=true', '-Dlogfile.name=ui.log', @@ -363,7 +363,7 @@ def test_logviewer_command(self): self.java_cmd, '-server', '-Ddaemon.name=logviewer', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/lib-webapp:' + self.storm_dir + '/conf', '-Djava.deserialization.disabled=true', '-Dlogfile.name=logviewer.log', @@ -380,7 +380,7 @@ def test_drpc_command(self): self.java_cmd, '-server', '-Ddaemon.name=drpc', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/lib-webapp:' + self.storm_dir + '/conf', '-Djava.deserialization.disabled=true', '-Dlogfile.name=drpc.log', @@ -397,7 +397,7 @@ def test_drpc_client_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.BasicDrpcClient', 'exclaim', 'a', 'exclaim', 'b', 'test', 'bar' ]) @@ -409,7 +409,7 @@ def test_drpc_client_command(self): self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + + self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.BasicDrpcClient', '-f', 'exclaim', 'a', 'b' ]) @@ -422,7 +422,7 @@ def test_healthcheck_command(self): self.java_cmd, [ self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", '-Djava.library.path=', - '-Dstorm.conf.file=', '-cp', self.storm_dir + '/*:' + self.storm_dir + '/lib:' + + '-Dstorm.conf.file=', '-cp', self.storm_dir + '/*:' + self.storm_dir + '/lib-common:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + self.storm_dir + '/extlib-daemon:' + self.storm_dir + '/conf:' + self.storm_dir + '/bin', 'org.apache.storm.command.HealthCheck' ]) From 6e35c8a1a205f39b757ea2d0493428cf40037169 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Thu, 9 Jul 2026 20:44:45 +0200 Subject: [PATCH 7/8] build: add a lean "storm-lite" distribution alongside the full one The full distribution keeps bundling storm-autocreds (Hadoop/HBase) and storm-kafka-monitor (Kafka client) so existing and air-gapped deployments are unaffected. A second "storm-lite" artifact omits those jars and ships only their READMEs, for users who do not need them and want a much smaller download. The shared assembly content moves into a common.xml component descriptor, so binary.xml (full) and binary-lite.xml differ only in the optional plugin jars. The lite artifact is attached with the "lite" Maven classifier and is signed by the existing GPG plugin. Both distributions keep the lib-common de-duplication. Measured (3.0.0-SNAPSHOT, tar.gz): baseline 367 MB, full now 303 MB (-64 MB via lib-common), lite 196 MB (-171 MB, ~47%). RELEASING.md documents the new artifact and the extra signing/checksum steps. --- RELEASING.md | 31 ++ bin/storm-autocreds-fetch | 6 +- bin/storm-kafka-monitor | 2 +- bin/storm-kafka-monitor-fetch | 8 +- external/storm-autocreds/README.md | 27 +- external/storm-kafka-monitor/README.md | 16 +- .../apache/storm/utils/TopologySpoutLag.java | 15 +- storm-dist/binary/final-package/pom.xml | 30 +- .../src/main/assembly/binary-lite.xml | 63 ++++ .../src/main/assembly/binary.xml | 330 ++---------------- .../src/main/assembly/common.xml | 328 +++++++++++++++++ storm-dist/binary/pom.xml | 2 + storm-dist/binary/storm-autocreds-bin/pom.xml | 63 ++++ .../src/main/assembly/storm-autocreds.xml | 33 ++ .../binary/storm-kafka-monitor-bin/pom.xml | 69 ++++ .../src/main/assembly/storm-kafka-monitor.xml | 33 ++ 16 files changed, 717 insertions(+), 339 deletions(-) create mode 100644 storm-dist/binary/final-package/src/main/assembly/binary-lite.xml create mode 100644 storm-dist/binary/final-package/src/main/assembly/common.xml create mode 100644 storm-dist/binary/storm-autocreds-bin/pom.xml create mode 100644 storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml create mode 100644 storm-dist/binary/storm-kafka-monitor-bin/pom.xml create mode 100644 storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml diff --git a/RELEASING.md b/RELEASING.md index 270437cddac..bcfe91baec0 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -101,9 +101,17 @@ In this way, you create a new release line and then you can create PATCH version pushd storm-dist/binary/final-package/target sha512sum apache-storm-2.8.1.zip > apache-storm-2.8.1.zip.sha512 sha512sum apache-storm-2.8.1.tar.gz > apache-storm-2.8.1.tar.gz.sha512 + sha512sum apache-storm-2.8.1-lite.zip > apache-storm-2.8.1-lite.zip.sha512 + sha512sum apache-storm-2.8.1-lite.tar.gz > apache-storm-2.8.1-lite.tar.gz.sha512 popd ``` + > **Note:** the binary build produces two distributions. The full one + > (`apache-storm-x.x.x.tar.gz` / `.zip`) bundles the optional `storm-autocreds` and + > `storm-kafka-monitor` jars, while the lean one (`apache-storm-x.x.x-lite.tar.gz` / `.zip`) + > omits them. Both must be signed, checksummed and staged for the release candidate. + > See [Storm lite distribution](#storm-lite-distribution) below. + 6. Create a directory in the dist svn repo for the release candidate: https://dist.apache.org/repos/dist/dev/storm/apache-storm-x.x.x-rcx 7. Before generating the release notes, please double check if all merged pull requests for the version being released are assigned the milestone in question. They won't be placed in the release notes otherwise. @@ -135,8 +143,31 @@ In this way, you create a new release line and then you can create PATCH version apache-storm-2.8.3-src.tar.gz apache-storm-2.8.3-src.zip apache-storm-2.8.3.tar.gz apache-storm-2.8.3.zip RELEASE_NOTES.html apache-storm-2.8.3-src.tar.gz.asc apache-storm-2.8.3-src.zip.asc apache-storm-2.8.3.tar.gz.asc apache-storm-2.8.3.zip.asc RELEASE_NOTES.html.asc apache-storm-2.8.3-src.tar.gz.sha512 apache-storm-2.8.3-src.zip.sha512 apache-storm-2.8.3.tar.gz.sha512 apache-storm-2.8.3.zip.sha512 RELEASE_NOTES.html.sha512 + apache-storm-2.8.3-lite.tar.gz apache-storm-2.8.3-lite.zip + apache-storm-2.8.3-lite.tar.gz.asc apache-storm-2.8.3-lite.zip.asc + apache-storm-2.8.3-lite.tar.gz.sha512 apache-storm-2.8.3-lite.zip.sha512 ``` +## Storm lite distribution + +Since 3.0.0 the binary build produces two distributions from `storm-dist/binary`: + +| Artifact | Contents | +|---|---| +| `apache-storm-x.x.x.tar.gz` / `.zip` | Full distribution. Bundles the `storm-autocreds` (Hadoop/HBase) and `storm-kafka-monitor` (Kafka client) jars. | +| `apache-storm-x.x.x-lite.tar.gz` / `.zip` | Lean distribution. Ships only the READMEs for those two plugins; everything else is identical. | + +Both are produced by the same `mvn package` run in `storm-dist/binary` (assembly +executions `bin` and `lite`), and both are automatically signed by the GPG plugin. +The lite artifact is attached with the `lite` Maven classifier. + +Release managers must sign, checksum and stage **both** distributions, and list +both in the release candidate VOTE mail so reviewers know which to verify. + +Users of the lite distribution can install the omitted plugins on demand with +`bin/storm-autocreds-fetch` and `bin/storm-kafka-monitor-fetch`, which resolve them +from Maven Central (or an internal mirror configured in `settings.xml`). + 10. Add and commit the files to SVN. This makes them available in the Apache staging repo. ```bash diff --git a/bin/storm-autocreds-fetch b/bin/storm-autocreds-fetch index d585be72b69..aabb98adbfe 100755 --- a/bin/storm-autocreds-fetch +++ b/bin/storm-autocreds-fetch @@ -20,9 +20,9 @@ # into the daemon classpath, so Nimbus/Supervisor can populate and renew HDFS # and HBase delegation tokens on a secure (Kerberos) cluster. # -# These jars are intentionally NOT bundled in the binary distribution to keep it -# small; only secure-Hadoop deployments need them. See -# external/storm-autocreds/README.md for details. +# These jars ship only in the full binary distribution; the lite distribution +# (apache-storm-x.x.x-lite.tar.gz) omits them because only secure-Hadoop +# deployments need them. See external/storm-autocreds/README.md for details. set -euo pipefail diff --git a/bin/storm-kafka-monitor b/bin/storm-kafka-monitor index a586c1d8d58..8f2d6b109e8 100755 --- a/bin/storm-kafka-monitor +++ b/bin/storm-kafka-monitor @@ -49,7 +49,7 @@ if [ -z "$JAVA_HOME" ]; then else JAVA="$JAVA_HOME/bin/java" fi -# The storm-kafka-monitor jars are not bundled in the distribution; they are fetched on demand. +# The storm-kafka-monitor jars ship only in the full distribution; on the lite one they are fetched on demand. KAFKA_MONITOR_LIB="$STORM_BASE_DIR/lib-tools/storm-kafka-monitor" if ! ls "$KAFKA_MONITOR_LIB"/*.jar >/dev/null 2>&1; then echo "storm-kafka-monitor is not installed (no jars in $KAFKA_MONITOR_LIB)." >&2 diff --git a/bin/storm-kafka-monitor-fetch b/bin/storm-kafka-monitor-fetch index 573312d87ef..43dd9022195 100755 --- a/bin/storm-kafka-monitor-fetch +++ b/bin/storm-kafka-monitor-fetch @@ -20,10 +20,10 @@ # into lib-tools/storm-kafka-monitor, enabling the "Kafka spout lag" display in # the Storm UI and the bin/storm-kafka-monitor command. # -# These jars are intentionally NOT bundled in the binary distribution to keep it -# small; they are only needed when running Kafka spouts and wanting lag info. The -# UI degrades gracefully when they are absent. See -# external/storm-kafka-monitor/README.md for details. +# These jars ship only in the full binary distribution; the lite distribution +# (apache-storm-x.x.x-lite.tar.gz) omits them because they are only needed when +# running Kafka spouts and wanting lag info. The UI degrades gracefully when they +# are absent. See external/storm-kafka-monitor/README.md for details. set -euo pipefail diff --git a/external/storm-autocreds/README.md b/external/storm-autocreds/README.md index 02459352994..4599724349b 100644 --- a/external/storm-autocreds/README.md +++ b/external/storm-autocreds/README.md @@ -12,20 +12,31 @@ HDFS or HBase cluster without distributing keytabs to every worker host. See `docs/SECURITY.md` ("Automatic Credentials Push and Renewal") for the full design. -## Why the jars are not bundled +## Which distribution has the jars? -Because these plugins run on the **daemon** classpath (Nimbus/Supervisor) and -pull in the full Hadoop and HBase client dependency trees, they are **not** -shipped inside the binary distribution — only secure-Hadoop deployments need -them, and bundling them would bloat the distribution for everyone. This is the -same convention used by the other `external/*` connectors. +These plugins run on the **daemon** classpath (Nimbus/Supervisor) and pull in the +full Hadoop and HBase client dependency trees, which is a large amount of weight +that only secure-Hadoop deployments need. Therefore: + +| Distribution | storm-autocreds jars | +|---|---| +| `apache-storm-x.x.x.tar.gz` (full) | bundled under `external/storm-autocreds` | +| `apache-storm-x.x.x-lite.tar.gz` (lite) | not bundled, README only | + +If you use the **lite** distribution, install the jars with the helper script +below (or just use the full distribution). ## Installing The plugins must be present on the **daemon** classpath, i.e. in -`$STORM_HOME/extlib-daemon` on Nimbus and the Supervisors. +`$STORM_HOME/extlib-daemon` on Nimbus and the Supervisors. With the full +distribution, copy them from `external/storm-autocreds`: + +```bash +cp $STORM_HOME/external/storm-autocreds/*.jar $STORM_HOME/extlib-daemon/ +``` -### Option 1 — use the helper script (recommended) +### Option 1 — use the helper script (recommended for the lite distribution) The distribution ships a helper that resolves `storm-autocreds` and its runtime dependencies from Maven Central and copies them into `extlib-daemon`: diff --git a/external/storm-kafka-monitor/README.md b/external/storm-kafka-monitor/README.md index 8e5bd37ef4e..24da7a9b80c 100644 --- a/external/storm-kafka-monitor/README.md +++ b/external/storm-kafka-monitor/README.md @@ -4,14 +4,20 @@ Tool to query kafka spout lags and show in Storm UI ## Installation -The storm-kafka-monitor jars (and their Kafka client dependencies) are **not** -bundled in the binary distribution to keep it small — they are only needed to -display Kafka spout lag in the UI or to run the `storm-kafka-monitor` command. +The storm-kafka-monitor jars (and their Kafka client dependencies) are only +needed to display Kafka spout lag in the UI or to run the `storm-kafka-monitor` +command, so they are bundled only in the full distribution: + +| Distribution | storm-kafka-monitor jars | +|---|---| +| `apache-storm-x.x.x.tar.gz` (full) | bundled under `lib-tools/storm-kafka-monitor`, works out of the box | +| `apache-storm-x.x.x-lite.tar.gz` (lite) | not bundled, README only | + The Storm UI degrades gracefully when they are absent (no lag is shown and a hint is logged once). -To enable it, install the jars on the UI host with the helper script, then -restart the UI: +With the **lite** distribution, install the jars on the UI host with the helper +script, then restart the UI: ```bash $STORM_HOME/bin/storm-kafka-monitor-fetch diff --git a/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java index a230e1e7a25..d6bc360522c 100644 --- a/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java +++ b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java @@ -54,9 +54,9 @@ public class TopologySpoutLag { BOOTSTRAP_CONFIG, SECURITY_PROTOCOL_CONFIG)); private static final Logger LOGGER = LoggerFactory.getLogger(TopologySpoutLag.class); - // The storm-kafka-monitor jars are not bundled in the binary distribution; operators install them - // on demand (bin/storm-kafka-monitor-fetch). Log the "not installed" hint at most once to avoid - // spamming the UI logs, which poll the lag endpoint periodically. + // The storm-kafka-monitor jars ship only in the full binary distribution; users of the lite + // distribution install them on demand (bin/storm-kafka-monitor-fetch). Log the "not installed" + // hint at most once to avoid spamming the UI logs, which poll the lag endpoint periodically. private static volatile boolean warnedMonitorMissing = false; public static Map> lag(StormTopology stormTopology, Map topologyConf) { @@ -76,8 +76,8 @@ public static Map> lag(StormTopology stormTopology, /** * Checks whether the storm-kafka-monitor jars (invoked by bin/storm-kafka-monitor) are present. - * They are not bundled in the binary distribution and are fetched on demand, so the UI must - * degrade gracefully when they are absent rather than failing the lag shell-out. + * They ship only in the full binary distribution and are fetched on demand on the lite one, so + * the UI must degrade gracefully when they are absent rather than failing the lag shell-out. * * @return true if the monitor appears installed, or if STORM_BASE_DIR is unknown (in which case * the legacy behavior of attempting the shell-out is preserved). @@ -191,8 +191,9 @@ private static Map getLagResultForKafka(String spoutId, SpoutSpe // if commands contains one or more null value, spout is compiled with lower version of storm-kafka-client if (!commands.contains(null) && !isKafkaMonitorInstalled()) { errorMsg = "Kafka spout lag monitoring is unavailable because the storm-kafka-monitor " - + "jars are not installed. They are no longer bundled in the binary distribution; " - + "run 'bin/storm-kafka-monitor-fetch' on the UI host (and restart the UI) to enable it."; + + "jars are not installed. They ship only in the full binary distribution; on the " + + "lite distribution run 'bin/storm-kafka-monitor-fetch' on the UI host (and restart " + + "the UI) to enable it."; if (!warnedMonitorMissing) { warnedMonitorMissing = true; LOGGER.info(errorMsg); diff --git a/storm-dist/binary/final-package/pom.xml b/storm-dist/binary/final-package/pom.xml index dce19eaf6ed..3155dea8fd5 100644 --- a/storm-dist/binary/final-package/pom.xml +++ b/storm-dist/binary/final-package/pom.xml @@ -114,21 +114,43 @@ org.apache.maven.plugins maven-assembly-plugin + + bin package single + + + ${project.basedir}/src/main/assembly/binary.xml + + false + + + + + lite + package + + single + + + + ${project.basedir}/src/main/assembly/binary-lite.xml + + true + true posix false - - ${project.basedir}/src/main/assembly/binary.xml - - false diff --git a/storm-dist/binary/final-package/src/main/assembly/binary-lite.xml b/storm-dist/binary/final-package/src/main/assembly/binary-lite.xml new file mode 100644 index 00000000000..ce0eb4c3b4a --- /dev/null +++ b/storm-dist/binary/final-package/src/main/assembly/binary-lite.xml @@ -0,0 +1,63 @@ + + + + + lite + + tar.gz + zip + + + + ${project.basedir}/src/main/assembly/common.xml + + + + + + ${project.basedir}/../../../external/storm-kafka-monitor + external/storm-kafka-monitor + + README.* + + + + + + ${project.basedir}/../../../external/storm-autocreds + external/storm-autocreds + + README.* + + + + diff --git a/storm-dist/binary/final-package/src/main/assembly/binary.xml b/storm-dist/binary/final-package/src/main/assembly/binary.xml index 5e62e2bc00d..32cd51031ab 100644 --- a/storm-dist/binary/final-package/src/main/assembly/binary.xml +++ b/storm-dist/binary/final-package/src/main/assembly/binary.xml @@ -15,6 +15,14 @@ limitations under the License. --> + @@ -24,220 +32,19 @@ zip - - - - ${project.build.directory}/staging/lib - lib - - *.jar - - - - ${project.build.directory}/staging/lib-common - lib-common - - *.jar - - - - ${project.basedir}/../storm-webapp-bin/target/webapp/webapp/ - . - - */** - - - - - ${project.basedir}/../../../bin - bin - - storm* - flight.bash - - unix - 0755 - - - - ${project.basedir}/../../../storm-webapp/target/classes/WEB-INF/ - public - - *.html - - - true - - - - ${project.basedir}/../../../storm-webapp/target/classes/WEB-INF/ - public - - */** - - - *.html - - - - - ${project.basedir}/../../../examples - examples - - **/target/** - **/dependency-reduced-pom.xml - **/*.iml - - - - ${project.basedir}/../../../storm-core/target/native/target/usr/local/bin/ - - worker-launcher - - bin - - - - - ${project.basedir}/../../../external/storm-kinesis - external/storm-kinesis - - README.* - - - - ${project.basedir}/../../../external/storm-hdfs - external/storm-hdfs - - README.* - - - - ${project.basedir}/../../../external/storm-jdbc - external/storm-jdbc - - README.* - - - - ${project.basedir}/../../../external/storm-eventhubs - external/storm-eventhubs - - README.* - - + + ${project.basedir}/src/main/assembly/common.xml + + + - ${project.basedir}/../../../external/flux - external/flux - - README.* - - - - ${project.basedir}/../../../external/storm-redis - external/storm-redis - - README.* - - - - ${project.basedir}/../../../external/storm-solr - external/storm-solr - - README.* - - - - ${project.basedir}/../../../external/storm-mqtt - external/storm-mqtt - - README.* - - - - ${project.basedir}/../../../external/storm-mongodb - external/storm-mongodb - - README.* - - - - ${project.basedir}/../../../external/storm-kafka-client - external/storm-kafka-client - - README.* - - - - ${project.basedir}/../../../external/storm-opentsdb - external/storm-opentsdb - - README.* - - - - ${project.basedir}/../../../external/storm-druid - external/storm-druid - - README.* - - - - ${project.basedir}/../../../external/storm-jms - external/storm-jms - - README.* - - - - ${project.basedir}/../../../external/storm-pmml - external/storm-pmml - - README.* - - - - ${project.basedir}/../../../external/storm-rocketmq - external/storm-rocketmq - - README.* - - - - - - - extlib - - - - - - - - extlib-daemon + ${project.basedir}/../storm-kafka-monitor-bin/target/kafka-monitor/kafka-monitor/lib-kafka-monitor + lib-tools/storm-kafka-monitor - + *jar - ${project.basedir}/../../../external/storm-kafka-monitor external/storm-kafka-monitor @@ -246,112 +53,21 @@ - + - ${project.basedir}/../../../external/storm-autocreds + ${project.basedir}/../storm-autocreds-bin/target/autocreds/autocreds/lib-autocreds external/storm-autocreds - README.* + *jar - - ${project.basedir}/../storm-submit-tools-bin/target/submit-tools/submit-tools/lib-submit-tools - lib-tools/submit-tools + ${project.basedir}/../../../external/storm-autocreds + external/storm-autocreds - *jar + README.* - - - ${project.basedir}/../../../licenses - licenses - - - - - - ${project.basedir}/../../../conf/storm.yaml.example - ./conf - storm.yaml - 0644 - - - ${project.basedir}/../../../conf/storm_env.ini - ./conf - storm_env.ini - 0644 - - - ${project.basedir}/../../../conf/storm-env.sh - ./conf - storm-env.sh - unix - 0755 - - - ${project.basedir}/../../../conf/storm-env.ps1 - ./conf - storm-env.ps1 - 0755 - - - ${project.basedir}/../../../conf/seccomp.json.example - /conf - seccomp.json.example - 0755 - - - ${project.basedir}/../../../VERSION - . - RELEASE - true - - - - ${project.basedir}/../../../log4j2/cluster.xml - ./log4j2 - 0644 - - - - ${project.basedir}/../../../log4j2/worker.xml - ./log4j2 - 0644 - - - - ${project.basedir}/../../../LICENSE-binary - LICENSE - . - - - - ${project.basedir}/../../../NOTICE-binary - NOTICE - . - - - - ${project.basedir}/../../../DEPENDENCY-LICENSES - . - - - - ${project.basedir}/../../../README.markdown - . - - - - ${project.basedir}/../../../SECURITY.md - . - - - diff --git a/storm-dist/binary/final-package/src/main/assembly/common.xml b/storm-dist/binary/final-package/src/main/assembly/common.xml new file mode 100644 index 00000000000..6b50cc8ada8 --- /dev/null +++ b/storm-dist/binary/final-package/src/main/assembly/common.xml @@ -0,0 +1,328 @@ + + + + + + + + ${project.build.directory}/staging/lib + lib + + *.jar + + + + ${project.build.directory}/staging/lib-common + lib-common + + *.jar + + + + ${project.basedir}/../storm-webapp-bin/target/webapp/webapp/ + . + + */** + + + + + ${project.basedir}/../../../bin + bin + + storm* + flight.bash + + unix + 0755 + + + + ${project.basedir}/../../../storm-webapp/target/classes/WEB-INF/ + public + + *.html + + + true + + + + ${project.basedir}/../../../storm-webapp/target/classes/WEB-INF/ + public + + */** + + + *.html + + + + + ${project.basedir}/../../../examples + examples + + **/target/** + **/dependency-reduced-pom.xml + **/*.iml + + + + ${project.basedir}/../../../storm-core/target/native/target/usr/local/bin/ + + worker-launcher + + bin + + + + + ${project.basedir}/../../../external/storm-kinesis + external/storm-kinesis + + README.* + + + + ${project.basedir}/../../../external/storm-hdfs + external/storm-hdfs + + README.* + + + + ${project.basedir}/../../../external/storm-jdbc + external/storm-jdbc + + README.* + + + + ${project.basedir}/../../../external/storm-eventhubs + external/storm-eventhubs + + README.* + + + + + ${project.basedir}/../../../external/flux + external/flux + + README.* + + + + ${project.basedir}/../../../external/storm-redis + external/storm-redis + + README.* + + + + ${project.basedir}/../../../external/storm-solr + external/storm-solr + + README.* + + + + ${project.basedir}/../../../external/storm-mqtt + external/storm-mqtt + + README.* + + + + ${project.basedir}/../../../external/storm-mongodb + external/storm-mongodb + + README.* + + + + ${project.basedir}/../../../external/storm-kafka-client + external/storm-kafka-client + + README.* + + + + ${project.basedir}/../../../external/storm-opentsdb + external/storm-opentsdb + + README.* + + + + ${project.basedir}/../../../external/storm-druid + external/storm-druid + + README.* + + + + ${project.basedir}/../../../external/storm-jms + external/storm-jms + + README.* + + + + ${project.basedir}/../../../external/storm-pmml + external/storm-pmml + + README.* + + + + ${project.basedir}/../../../external/storm-rocketmq + external/storm-rocketmq + + README.* + + + + + + + extlib + + + + + + + + extlib-daemon + + + + + + + ${project.basedir}/../storm-submit-tools-bin/target/submit-tools/submit-tools/lib-submit-tools + lib-tools/submit-tools + + *jar + + + + + ${project.basedir}/../../../licenses + licenses + + + + + + + ${project.basedir}/../../../conf/storm.yaml.example + ./conf + storm.yaml + 0644 + + + ${project.basedir}/../../../conf/storm_env.ini + ./conf + storm_env.ini + 0644 + + + ${project.basedir}/../../../conf/storm-env.sh + ./conf + storm-env.sh + unix + 0755 + + + ${project.basedir}/../../../conf/storm-env.ps1 + ./conf + storm-env.ps1 + 0755 + + + ${project.basedir}/../../../conf/seccomp.json.example + /conf + seccomp.json.example + 0755 + + + ${project.basedir}/../../../VERSION + . + RELEASE + true + + + + ${project.basedir}/../../../log4j2/cluster.xml + ./log4j2 + 0644 + + + + ${project.basedir}/../../../log4j2/worker.xml + ./log4j2 + 0644 + + + + ${project.basedir}/../../../LICENSE-binary + LICENSE + . + + + + ${project.basedir}/../../../NOTICE-binary + NOTICE + . + + + + ${project.basedir}/../../../DEPENDENCY-LICENSES + . + + + + ${project.basedir}/../../../README.markdown + . + + + + ${project.basedir}/../../../SECURITY.md + . + + + diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml index 3b58038d172..94209592e3c 100644 --- a/storm-dist/binary/pom.xml +++ b/storm-dist/binary/pom.xml @@ -47,7 +47,9 @@ storm-client-bin storm-webapp-bin + storm-autocreds-bin storm-submit-tools-bin + storm-kafka-monitor-bin final-package diff --git a/storm-dist/binary/storm-autocreds-bin/pom.xml b/storm-dist/binary/storm-autocreds-bin/pom.xml new file mode 100644 index 00000000000..8b42b902fff --- /dev/null +++ b/storm-dist/binary/storm-autocreds-bin/pom.xml @@ -0,0 +1,63 @@ + + + + 4.0.0 + + org.apache.storm + apache-storm-bin + 3.0.0-SNAPSHOT + + storm-autocreds-bin + pom + + + Storm Autocreds Binary + + + org.apache.storm + storm-autocreds + ${project.version} + + + + + autocreds + + + org.apache.maven.plugins + maven-assembly-plugin + + + prepare-package + + single + + + + + false + false + + ${project.basedir}/src/main/assembly/storm-autocreds.xml + + false + + + + + diff --git a/storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml b/storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml new file mode 100644 index 00000000000..1cd914cb7b7 --- /dev/null +++ b/storm-dist/binary/storm-autocreds-bin/src/main/assembly/storm-autocreds.xml @@ -0,0 +1,33 @@ + + + + storm-autocreds-bin + + dir + + + + false + lib-autocreds + false + + + diff --git a/storm-dist/binary/storm-kafka-monitor-bin/pom.xml b/storm-dist/binary/storm-kafka-monitor-bin/pom.xml new file mode 100644 index 00000000000..4d9e569a95a --- /dev/null +++ b/storm-dist/binary/storm-kafka-monitor-bin/pom.xml @@ -0,0 +1,69 @@ + + + + 4.0.0 + + org.apache.storm + apache-storm-bin + 3.0.0-SNAPSHOT + + storm-kafka-monitor-bin + pom + + + Storm Kafka Monitor Binary + + + org.apache.storm + storm-kafka-monitor + ${project.version} + + + + org.apache.kafka + kafka-clients + compile + + + + + kafka-monitor + + + org.apache.maven.plugins + maven-assembly-plugin + + + prepare-package + + single + + + + + false + false + + ${project.basedir}/src/main/assembly/storm-kafka-monitor.xml + + false + + + + + diff --git a/storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml b/storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml new file mode 100644 index 00000000000..021b472c87f --- /dev/null +++ b/storm-dist/binary/storm-kafka-monitor-bin/src/main/assembly/storm-kafka-monitor.xml @@ -0,0 +1,33 @@ + + + + storm-kafka-monitor-bin + + dir + + + + false + lib-kafka-monitor + false + + + From b54c02872b6cdfaa8a8cbf3720df5a5a860d887a Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Thu, 9 Jul 2026 21:22:52 +0200 Subject: [PATCH 8/8] build: keep tooling on the full distribution now that storm-lite is built The binary build produces two zips/tarballs, which broke consumers that assumed exactly one artifact: - integration-test/run-it.sh failed its "expected exactly one zip file" guard. - integration-test/config/Vagrantfile raised "Expected one storm-binary". Both now ignore apache-storm-*-lite.zip and run against the full distribution. The cluster Dockerfile ADDs the full tarball by name, so its .dockerignore also excludes the lite tarball to keep it out of the build context. --- dev-tools/cluster/Dockerfile.dockerignore | 3 +++ integration-test/config/Vagrantfile | 4 +++- integration-test/run-it.sh | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dev-tools/cluster/Dockerfile.dockerignore b/dev-tools/cluster/Dockerfile.dockerignore index a12edc5157b..0780dbbc195 100644 --- a/dev-tools/cluster/Dockerfile.dockerignore +++ b/dev-tools/cluster/Dockerfile.dockerignore @@ -20,3 +20,6 @@ # (which also holds the .zip and the extracted distribution). * !apache-storm-*.tar.gz +# The build ADDs the full distribution; keep the lean "storm-lite" tarball out of +# the context so it does not add ~200 MB for nothing. +apache-storm-*-lite.tar.gz diff --git a/integration-test/config/Vagrantfile b/integration-test/config/Vagrantfile index ab6e774891c..f09721190b0 100644 --- a/integration-test/config/Vagrantfile +++ b/integration-test/config/Vagrantfile @@ -20,7 +20,9 @@ require 'uri' # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! VAGRANTFILE_API_VERSION = "2" STORM_BOX_TYPE = "ubuntu/xenial64" -STORM_ZIP = Dir.glob("../../storm-dist/binary/**/*.zip") +# The binary build also produces the lean "storm-lite" distribution; the integration +# tests run against the full one, so ignore apache-storm-*-lite.zip here. +STORM_ZIP = Dir.glob("../../storm-dist/binary/**/*.zip").reject { |f| f.end_with?("-lite.zip") } if(STORM_ZIP.length != 1) raise "Expected one storm-binary found: " + STORM_ZIP.join(",") + ". Did you run : cd ${STORM_SRC_DIR}/storm-dist/binary && mvn clean package -Dgpg.skip=true" end diff --git a/integration-test/run-it.sh b/integration-test/run-it.sh index 3b1f10e8b07..a411456ff13 100755 --- a/integration-test/run-it.sh +++ b/integration-test/run-it.sh @@ -62,10 +62,12 @@ else if [[ "${USER}" == "github" ]]; then ( cd "${STORM_SRC_DIR}/storm-dist/binary" && mvn clean package -Dgpg.skip=true ) fi - (( $(find "${STORM_SRC_DIR}/storm-dist/binary" -iname 'apache-storm*.zip' | wc -l) == 1 )) || die "expected exactly one zip file, did you run: cd ${STORM_SRC_DIR}/storm-dist/binary && mvn clean package -Dgpg.skip=true" + # The binary build also produces the lean "storm-lite" distribution; the integration + # tests run against the full one, so ignore apache-storm-*-lite.zip here. + (( $(find "${STORM_SRC_DIR}/storm-dist/binary" -iname 'apache-storm*.zip' -not -iname '*-lite.zip' | wc -l) == 1 )) || die "expected exactly one zip file, did you run: cd ${STORM_SRC_DIR}/storm-dist/binary && mvn clean package -Dgpg.skip=true" fi -storm_binary_zip=$(find "${STORM_SRC_DIR}/storm-dist" -iname '*.zip') +storm_binary_zip=$(find "${STORM_SRC_DIR}/storm-dist" -iname 'apache-storm*.zip' -not -iname '*-lite.zip') storm_binary_name=$(basename "${storm_binary_zip}") export STORM_VERSION=$(grep -oPe '\d.*(?=.zip)' <<<"${storm_binary_name}") echo "Using storm version:" ${STORM_VERSION}