diff --git a/CHANGELOG.md b/CHANGELOG.md index caef2657e..acee9a8e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ All notable changes to this project will be documented in this file. - superset: Pin npm to `10.9.9` instead of installing the latest version with `nvm install --latest-npm` ([#1682]). - opa: update golang to 1.27.1 ([#1697]). - spark: Pin pyarrow below 26 to satisfy scikit-learn dependency on NumPy 1.x ([#1695]). +- spark-connect-client: Replace hard-coded Python dependency versions with image dependencies and bump them for Spark 4.2.0 ([#1713]). ### Fixed @@ -165,6 +166,7 @@ All notable changes to this project will be documented in this file. [#1709]: https://github.com/stackabletech/docker-images/pull/1709 [#1710]: https://github.com/stackabletech/docker-images/pull/1710 [#1712]: https://github.com/stackabletech/docker-images/pull/1712 +[#1713]: https://github.com/stackabletech/docker-images/pull/1713 ## [26.7.0] - 2026-07-21 diff --git a/spark-connect-client/Dockerfile b/spark-connect-client/Dockerfile index 86d093812..6655b1447 100644 --- a/spark-connect-client/Dockerfile +++ b/spark-connect-client/Dockerfile @@ -5,6 +5,10 @@ FROM local-image/spark-k8s AS spark-builder ARG PRODUCT_VERSION ARG PYTHON_VERSION +ARG JUPYTERLAB_VERSION +ARG SCIKIT_LEARN_VERSION +ARG MATPLOTLIB_VERSION +ARG PYARROW_MAX_VERSION ARG RELEASE_VERSION ARG STACKABLE_USER_UID @@ -41,13 +45,11 @@ USER ${STACKABLE_USER_UID} # Packages are intentionally installed in "user mode" to reduce the container attack surface. # - pyspark[connect] = spark connect client libs # - jupyterlab = notebook client used in demos -# - pyarrow < 26 = pyarrow 26 requires NumPy >= 2, but scikit-learn 1.3.1 requires -# NumPy 1.x. pyarrow does not declare NumPy as a dependency, thus pip cannot detect the conflict. RUN pip install --no-cache-dir --user \ "pyspark[connect]==${PRODUCT_VERSION}" \ - "pyarrow<26" \ - "jupyterlab==4.4.1" \ - "scikit-learn==1.3.1" \ - "matplotlib==3.10.1" + "pyarrow<${PYARROW_MAX_VERSION}" \ + "jupyterlab==${JUPYTERLAB_VERSION}" \ + "scikit-learn==${SCIKIT_LEARN_VERSION}" \ + "matplotlib==${MATPLOTLIB_VERSION}" WORKDIR /stackable diff --git a/spark-connect-client/boil-config.toml b/spark-connect-client/boil-config.toml index 3577b28d9..17b1c01f6 100644 --- a/spark-connect-client/boil-config.toml +++ b/spark-connect-client/boil-config.toml @@ -7,6 +7,10 @@ java-base = "21" [versions."4.1.2".build-arguments] python-version = "3.12" +jupyterlab-version = "4.4.1" +scikit-learn-version = "1.3.1" +matplotlib-version = "3.10.1" +pyarrow-max-version = "26" [versions."4.1.3".local-images] spark-k8s = "4.1.3" @@ -14,10 +18,25 @@ java-base = "21" [versions."4.1.3".build-arguments] python-version = "3.12" +jupyterlab-version = "4.4.1" +scikit-learn-version = "1.3.1" +matplotlib-version = "3.10.1" +pyarrow-max-version = "26" [versions."4.2.0".local-images] spark-k8s = "4.2.0" java-base = "21" +# Use this command to fetch the latest versions of these dependencies: +# +# printf 'pyspark[connect]==4.2.0\njupyterlab\nscikit-learn\nmatplotlib\npyarrow\n' \ +# | uv pip compile - --python-version 3.14 --no-header --no-annotate \ +# | grep -iE '^(pyspark|jupyterlab|scikit-learn|matplotlib|pyarrow|numpy|pandas)==' +# +# Change the pyspark[connect] version accordingly. [versions."4.2.0".build-arguments] -python-version = "3.12" +python-version = "3.14" +jupyterlab-version = "4.6.4" +scikit-learn-version = "1.9.1" +matplotlib-version = "3.11.2" +pyarrow-max-version = "27"