Reduce binary distribution size: unbundle optional Hadoop/Kafka deps + de-duplicate shared jars

RELEASING.md

@@ -101,9 +101,17 @@ In this way, you create a new release line and then you can create PATCH version
    pushd storm-dist/binary/final-package/target
    sha512sum apache-storm-2.8.1.zip > apache-storm-2.8.1.zip.sha512
    sha512sum apache-storm-2.8.1.tar.gz > apache-storm-2.8.1.tar.gz.sha512
+   sha512sum apache-storm-2.8.1-lite.zip > apache-storm-2.8.1-lite.zip.sha512
+   sha512sum apache-storm-2.8.1-lite.tar.gz > apache-storm-2.8.1-lite.tar.gz.sha512
    popd
    ```
 
+   > **Note:** the binary build produces two distributions. The full one
+   > (`apache-storm-x.x.x.tar.gz` / `.zip`) bundles the optional `storm-autocreds` and
+   > `storm-kafka-monitor` jars, while the lean one (`apache-storm-x.x.x-lite.tar.gz` / `.zip`)
+   > omits them. Both must be signed, checksummed and staged for the release candidate.
+   > See [Storm lite distribution](#storm-lite-distribution) below.
+
 6. Create a directory in the dist svn repo for the release candidate: https://dist.apache.org/repos/dist/dev/storm/apache-storm-x.x.x-rcx
 
 7. Before generating the release notes, please double check if all merged pull requests for the version being released are assigned the milestone in question. They won't be placed in the release notes otherwise.
@@ -135,8 +143,31 @@ In this way, you create a new release line and then you can create PATCH version
    apache-storm-2.8.3-src.tar.gz         apache-storm-2.8.3-src.zip         apache-storm-2.8.3.tar.gz         apache-storm-2.8.3.zip         RELEASE_NOTES.html
    apache-storm-2.8.3-src.tar.gz.asc     apache-storm-2.8.3-src.zip.asc     apache-storm-2.8.3.tar.gz.asc     apache-storm-2.8.3.zip.asc     RELEASE_NOTES.html.asc
    apache-storm-2.8.3-src.tar.gz.sha512  apache-storm-2.8.3-src.zip.sha512  apache-storm-2.8.3.tar.gz.sha512  apache-storm-2.8.3.zip.sha512  RELEASE_NOTES.html.sha512
+   apache-storm-2.8.3-lite.tar.gz        apache-storm-2.8.3-lite.zip
+   apache-storm-2.8.3-lite.tar.gz.asc    apache-storm-2.8.3-lite.zip.asc
+   apache-storm-2.8.3-lite.tar.gz.sha512 apache-storm-2.8.3-lite.zip.sha512
    ```
 
+## Storm lite distribution
+
+Since 3.0.0 the binary build produces two distributions from `storm-dist/binary`:
+
+| Artifact | Contents |
+|---|---|
+| `apache-storm-x.x.x.tar.gz` / `.zip` | Full distribution. Bundles the `storm-autocreds` (Hadoop/HBase) and `storm-kafka-monitor` (Kafka client) jars. |
+| `apache-storm-x.x.x-lite.tar.gz` / `.zip` | Lean distribution. Ships only the READMEs for those two plugins; everything else is identical. |
+
+Both are produced by the same `mvn package` run in `storm-dist/binary` (assembly
+executions `bin` and `lite`), and both are automatically signed by the GPG plugin.
+The lite artifact is attached with the `lite` Maven classifier.
+
+Release managers must sign, checksum and stage **both** distributions, and list
+both in the release candidate VOTE mail so reviewers know which to verify.
+
+Users of the lite distribution can install the omitted plugins on demand with
+`bin/storm-autocreds-fetch` and `bin/storm-kafka-monitor-fetch`, which resolve them
+from Maven Central (or an internal mirror configured in `settings.xml`).
+
 10. Add and commit the files to SVN. This makes them available in the Apache staging repo.
 
     ```bash

bin/storm-autocreds-fetch

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Fetch the storm-autocreds plugin and its (Hadoop/HBase) runtime dependencies
+# into the daemon classpath, so Nimbus/Supervisor can populate and renew HDFS
+# and HBase delegation tokens on a secure (Kerberos) cluster.
+#
+# These jars ship only in the full binary distribution; the lite distribution
+# (apache-storm-x.x.x-lite.tar.gz) omits them because only secure-Hadoop
+# deployments need them. See external/storm-autocreds/README.md for details.
+
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage: storm-autocreds-fetch [options] [-- <extra maven args>]
+
+Resolves org.apache.storm:storm-autocreds and its runtime dependencies from a
+Maven repository (Maven Central by default) and copies them into the Storm
+daemon classpath directory (extlib-daemon).
+
+Options:
+  --version <ver>   Storm version to fetch (default: read from $STORM_HOME/RELEASE)
+  --dest <dir>      Target directory (default: $STORM_HOME/extlib-daemon)
+  -h, --help        Show this help
+
+Any arguments after "--" are passed through to Maven, e.g. to use an internal
+mirror or an offline local repository:
+  storm-autocreds-fetch -- -s /path/settings.xml
+  storm-autocreds-fetch -- -Dmaven.repo.local=/path/to/offline-repo -o
+EOF
+}
+
+# Resolve symlinks so STORM_HOME is correct even when invoked via a link.
+PRG="${0}"
+while [ -h "${PRG}" ]; do
+  ls=$(ls -ld "${PRG}")
+  link=$(expr "${ls}" : '.*-> \(.*\)$')
+  if expr "${link}" : '/.*' > /dev/null; then
+    PRG="${link}"
+  else
+    PRG="$(dirname "${PRG}")/${link}"
+  fi
+done
+STORM_BIN_DIR=$(dirname "${PRG}")
+STORM_HOME=$(cd "${STORM_BIN_DIR}/.." && pwd)
+
+VERSION=""
+DEST=""
+MVN_ARGS=()
+while [ $# -gt 0 ]; do
+  case "${1}" in
+    --version) VERSION="${2}"; shift 2 ;;
+    --dest)    DEST="${2}"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    --)        shift; MVN_ARGS=("$@"); break ;;
+    *)         echo "Unknown option: ${1}" >&2; usage; exit 1 ;;
+  esac
+done
+
+if [ -z "${VERSION}" ]; then
+  if [ -f "${STORM_HOME}/RELEASE" ]; then
+    VERSION=$(tr -d '[:space:]' < "${STORM_HOME}/RELEASE")
+  fi
+fi
+if [ -z "${VERSION}" ]; then
+  echo "Error: could not determine Storm version. Pass --version <ver>." >&2
+  exit 1
+fi
+
+if [ -z "${DEST}" ]; then
+  DEST="${STORM_HOME}/extlib-daemon"
+fi
+
+MVN="${MAVEN_HOME:+${MAVEN_HOME}/bin/}mvn"
+if ! command -v "${MVN}" > /dev/null 2>&1; then
+  echo "Error: '${MVN}' not found on PATH. Install Apache Maven or set MAVEN_HOME." >&2
+  exit 1
+fi
+
+mkdir -p "${DEST}"
+
+# Use a throwaway POM that depends on storm-autocreds; copy-dependencies then
+# pulls the exact runtime closure (honoring the exclusions declared in the
+# published storm-autocreds POM). storm-client is 'provided' there and is
+# correctly skipped, since it already ships in lib/.
+TMP_DIR=$(mktemp -d)
+trap 'rm -rf "${TMP_DIR}"' EXIT
+cat > "${TMP_DIR}/pom.xml" <<EOF
+<project xmlns="http://maven.apache.org/POM/4.0.0">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.storm.tools</groupId>
+  <artifactId>storm-autocreds-fetch</artifactId>
+  <version>${VERSION}</version>
+  <packaging>pom</packaging>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.storm</groupId>
+      <artifactId>storm-autocreds</artifactId>
+      <version>${VERSION}</version>
+    </dependency>
+  </dependencies>
+</project>
+EOF
+
+echo "Fetching org.apache.storm:storm-autocreds:${VERSION} (runtime closure) into:"
+echo "  ${DEST}"
+"${MVN}" -q -f "${TMP_DIR}/pom.xml" \
+  org.apache.maven.plugins:maven-dependency-plugin:copy-dependencies \
+  -DincludeScope=runtime \
+  -DoutputDirectory="${DEST}" \
+  ${MVN_ARGS[@]+"${MVN_ARGS[@]}"}
+
+echo "Done. ${DEST} now contains:"
+ls -1 "${DEST}" | sed 's/^/  /'
+echo
+echo "Restart the Storm daemons (Nimbus, Supervisor) to pick up the new classpath,"
+echo "then configure the autocreds plugins in storm.yaml. See"
+echo "external/storm-autocreds/README.md for the required settings."

bin/storm-kafka-monitor

@@ -49,4 +49,11 @@ if [ -z "$JAVA_HOME" ]; then
 else
   JAVA="$JAVA_HOME/bin/java"
 fi
-exec $JAVA $STORM_JAAS_CONF_PARAM $STORM_JAR_JVM_OPTS -cp "$STORM_BASE_DIR/lib-tools/storm-kafka-monitor/*" org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@"
+# The storm-kafka-monitor jars ship only in the full distribution; on the lite one they are fetched on demand.
+KAFKA_MONITOR_LIB="$STORM_BASE_DIR/lib-tools/storm-kafka-monitor"
+if ! ls "$KAFKA_MONITOR_LIB"/*.jar >/dev/null 2>&1; then
+  echo "storm-kafka-monitor is not installed (no jars in $KAFKA_MONITOR_LIB)." >&2
+  echo "Run '$STORM_BIN_DIR/storm-kafka-monitor-fetch' to download it, then retry." >&2
+  exit 1
+fi
+exec $JAVA $STORM_JAAS_CONF_PARAM $STORM_JAR_JVM_OPTS -cp "$KAFKA_MONITOR_LIB/*" org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@"

bin/storm-kafka-monitor-fetch

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Fetch the storm-kafka-monitor tool and its (Kafka client) runtime dependencies
+# into lib-tools/storm-kafka-monitor, enabling the "Kafka spout lag" display in
+# the Storm UI and the bin/storm-kafka-monitor command.
+#
+# These jars ship only in the full binary distribution; the lite distribution
+# (apache-storm-x.x.x-lite.tar.gz) omits them because they are only needed when
+# running Kafka spouts and wanting lag info. The UI degrades gracefully when they
+# are absent. See external/storm-kafka-monitor/README.md for details.
+
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage: storm-kafka-monitor-fetch [options] [-- <extra maven args>]
+
+Resolves org.apache.storm:storm-kafka-monitor and its runtime dependencies from
+a Maven repository (Maven Central by default) and copies them into
+lib-tools/storm-kafka-monitor.
+
+Options:
+  --version <ver>   Storm version to fetch (default: read from $STORM_HOME/RELEASE)
+  --dest <dir>      Target directory (default: $STORM_HOME/lib-tools/storm-kafka-monitor)
+  -h, --help        Show this help
+
+Any arguments after "--" are passed through to Maven, e.g. to use an internal
+mirror or an offline local repository:
+  storm-kafka-monitor-fetch -- -s /path/settings.xml
+  storm-kafka-monitor-fetch -- -Dmaven.repo.local=/path/to/offline-repo -o
+EOF
+}
+
+# Resolve symlinks so STORM_HOME is correct even when invoked via a link.
+PRG="${0}"
+while [ -h "${PRG}" ]; do
+  ls=$(ls -ld "${PRG}")
+  link=$(expr "${ls}" : '.*-> \(.*\)$')
+  if expr "${link}" : '/.*' > /dev/null; then
+    PRG="${link}"
+  else
+    PRG="$(dirname "${PRG}")/${link}"
+  fi
+done
+STORM_BIN_DIR=$(dirname "${PRG}")
+STORM_HOME=$(cd "${STORM_BIN_DIR}/.." && pwd)
+
+VERSION=""
+DEST=""
+MVN_ARGS=()
+while [ $# -gt 0 ]; do
+  case "${1}" in
+    --version) VERSION="${2}"; shift 2 ;;
+    --dest)    DEST="${2}"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    --)        shift; MVN_ARGS=("$@"); break ;;
+    *)         echo "Unknown option: ${1}" >&2; usage; exit 1 ;;
+  esac
+done
+
+if [ -z "${VERSION}" ]; then
+  if [ -f "${STORM_HOME}/RELEASE" ]; then
+    VERSION=$(tr -d '[:space:]' < "${STORM_HOME}/RELEASE")
+  fi
+fi
+if [ -z "${VERSION}" ]; then
+  echo "Error: could not determine Storm version. Pass --version <ver>." >&2
+  exit 1
+fi
+
+if [ -z "${DEST}" ]; then
+  DEST="${STORM_HOME}/lib-tools/storm-kafka-monitor"
+fi
+
+MVN="${MAVEN_HOME:+${MAVEN_HOME}/bin/}mvn"
+if ! command -v "${MVN}" > /dev/null 2>&1; then
+  echo "Error: '${MVN}' not found on PATH. Install Apache Maven or set MAVEN_HOME." >&2
+  exit 1
+fi
+
+mkdir -p "${DEST}"
+
+# Use a throwaway POM that depends on storm-kafka-monitor; copy-dependencies then
+# pulls the exact runtime closure. The artifact itself is a direct dependency and
+# is therefore copied too.
+TMP_DIR=$(mktemp -d)
+trap 'rm -rf "${TMP_DIR}"' EXIT
+cat > "${TMP_DIR}/pom.xml" <<EOF
+<project xmlns="http://maven.apache.org/POM/4.0.0">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.storm.tools</groupId>
+  <artifactId>storm-kafka-monitor-fetch</artifactId>
+  <version>${VERSION}</version>
+  <packaging>pom</packaging>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.storm</groupId>
+      <artifactId>storm-kafka-monitor</artifactId>
+      <version>${VERSION}</version>
+    </dependency>
+  </dependencies>
+</project>
+EOF
+
+echo "Fetching org.apache.storm:storm-kafka-monitor:${VERSION} (runtime closure) into:"
+echo "  ${DEST}"
+"${MVN}" -q -f "${TMP_DIR}/pom.xml" \
+  org.apache.maven.plugins:maven-dependency-plugin:copy-dependencies \
+  -DincludeScope=runtime \
+  -DoutputDirectory="${DEST}" \
+  ${MVN_ARGS[@]+"${MVN_ARGS[@]}"}
+
+echo "Done. ${DEST} now contains:"
+ls -1 "${DEST}" | sed 's/^/  /'
+echo
+echo "Restart the Storm UI to enable Kafka spout lag display, or run"
+echo "bin/storm-kafka-monitor directly. See external/storm-kafka-monitor/README.md."

bin/storm.py

@@ -107,6 +107,7 @@ def confvalue(name, storm_config_opts, extrapaths, overriding_conf_file=None, da
 
 def get_classpath(extrajars, daemon=True, client=False):
     ret = get_wildcard_dir(STORM_DIR)
+    ret.extend(get_wildcard_dir(STORM_COMMON_LIB_DIR))
     if client:
         ret.extend(get_wildcard_dir(STORM_WORKER_LIB_DIR))
     else:
@@ -125,9 +126,9 @@ def get_classpath(extrajars, daemon=True, client=False):
 def init_storm_env(within_unittest=False):
 
     global NORMAL_CLASS_PATH, STORM_DIR, USER_CONF_DIR, STORM_CONF_DIR, STORM_WORKER_LIB_DIR, STORM_LIB_DIR,\
-        STORM_TOOLS_LIB_DIR, STORM_WEBAPP_LIB_DIR, STORM_BIN_DIR, STORM_LOG4J2_CONF_DIR, STORM_SUPERVISOR_LOG_FILE,\
-        CLUSTER_CONF_DIR, JAR_JVM_OPTS, JAVA_HOME, JAVA_CMD, CONF_FILE, STORM_EXT_CLASSPATH, \
-        STORM_EXT_CLASSPATH_DAEMON, LOCAL_TTL_DEFAULT
+        STORM_COMMON_LIB_DIR, STORM_TOOLS_LIB_DIR, STORM_WEBAPP_LIB_DIR, STORM_BIN_DIR, STORM_LOG4J2_CONF_DIR,\
+        STORM_SUPERVISOR_LOG_FILE, CLUSTER_CONF_DIR, JAR_JVM_OPTS, JAVA_HOME, JAVA_CMD, CONF_FILE, \
+        STORM_EXT_CLASSPATH, STORM_EXT_CLASSPATH_DAEMON, LOCAL_TTL_DEFAULT
 
     NORMAL_CLASS_PATH = cygpath if sys.platform == 'cygwin' else identity
     STORM_DIR = os.sep.join(os.path.realpath( __file__ ).split(os.sep)[:-2])
@@ -141,6 +142,10 @@ def init_storm_env(within_unittest=False):
 
     STORM_WORKER_LIB_DIR = os.path.join(STORM_DIR, "lib-worker")
     STORM_LIB_DIR = os.path.join(STORM_DIR, "lib")
+    # Jars shared by the daemon (lib) and worker (lib-worker) classpaths are de-duplicated into
+    # lib-common to keep the distribution small. It is added to both classpaths; absent in older
+    # layouts, in which case it contributes nothing.
+    STORM_COMMON_LIB_DIR = os.path.join(STORM_DIR, "lib-common")
 
     STORM_TOOLS_LIB_DIR = os.path.join(STORM_DIR, "lib-tools")
     STORM_WEBAPP_LIB_DIR = os.path.join(STORM_DIR, "lib-webapp")

bin/test_storm.py

@@ -66,6 +66,19 @@ def test_get_classpath(self):
         expected = ":".join(extrajars)
         self.assertEqual(s[-len(expected):], expected)
 
+    def test_get_classpath_includes_lib_common(self):
+        extrajars = []
+        # When lib-common exists, it is included on both the daemon and the client classpaths.
+        storm.STORM_COMMON_LIB_DIR = storm.STORM_BIN_DIR
+        expected = os.path.join(storm.STORM_BIN_DIR, "*")
+        for client in (True, False):
+            cp = storm.get_classpath(extrajars, daemon=True, client=client)
+            self.assertIn(expected, cp.split(os.pathsep))
+        # When it does not exist, it contributes nothing (backward compatible with older layouts).
+        storm.STORM_COMMON_LIB_DIR = os.path.join(storm.STORM_DIR, "no-such-lib-common")
+        cp = storm.get_classpath(extrajars, daemon=True, client=False)
+        self.assertNotIn(os.path.join(storm.STORM_COMMON_LIB_DIR, "*"), cp.split(os.pathsep))
+
     def test_resolve_dependencies(self):
         artifacts = "org.apache.commons.commons-api"
         artifact_repositories = "maven-central"

dev-tools/cluster/Dockerfile.dockerignore

@@ -20,3 +20,6 @@
 # (which also holds the .zip and the extracted distribution).
 *
 !apache-storm-*.tar.gz
+# The build ADDs the full distribution; keep the lean "storm-lite" tarball out of
+# the context so it does not add ~200 MB for nothing.
+apache-storm-*-lite.tar.gz