diff --git a/Pipfile b/Pipfile index c28be92..3e83267 100644 --- a/Pipfile +++ b/Pipfile @@ -7,7 +7,31 @@ name = "pypi" # py4j is needed for connecting to the JVM py4j = "==0.10.9.7" # https://spark.apache.org/docs/latest/api/python/getting_started/install.html#dependencies # pyspark is needed for working with Spark -pyspark = "==3.5.1" # should match the version of spark we use for testing +# Must match the Spark JVM the tests actually run against: spark.Dockerfile is +# icanbwell/helix.spark:3.5.5.0-slim, which builds on `spark:3.5.5-java17-python3` +# and sets SPARK_HOME=/opt/spark. pyspark honours SPARK_HOME over its bundled +# jars, so CI's Catalyst is Spark 3.5.5 either way -- pinning 3.5.1 here created +# a Python/JVM skew rather than preventing one. Also closes CVE-2025-55039 +# (patched in 3.5.2). +# +# PR #180 (LR-503) set this to 3.5.1 to avoid a helix.pipelines conflict. That +# is safe to undo because #180 never reached a consumer: it changed only Pipfile +# and Pipfile.lock, and neither is published (verified absent from the released +# sdist and wheel). SAM 3.0.6 was published 8 minutes after #180 merged and +# still declared pyspark==3.5.5. helix.pipelines is exact-pinned to +# sparkautomapper==3.0.2, an immutable artifact this cannot alter, and it pins +# pyspark==3.5.1 in its own pyproject.toml anyway. +# +# Do NOT justify this by "pyspark is in extras_require so consumers never see +# it" -- that only became true in 3.0.4 (PR #178). The 3.0.2 wheel that +# helix.pipelines consumes declares `Requires-Dist: pyspark==3.5.1` +# unconditionally. +# +# WARNING: `make pipenv-setup` (run by `make update`) flattens [packages] into +# setup.py's install_requires, which WOULD push this version to consumers and +# undo #178. That is how 3.0.3 shipped an unconditional pyspark==3.5.5. No CI +# job runs it, so keep setup.py hand-maintained. +pyspark = "==3.5.5" # should match the version of spark we use for testing logger = ">=1.4" sparkdataframecomparer = "==2.0.16" Deprecated = ">=1.2.12" diff --git a/Pipfile.lock b/Pipfile.lock index 72da67c..5f323bd 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ad3c856ce083eb9e226ce8300323063c4961ee862468d32cf9e5372742dbe373" + "sha256": "a67976abb0fcefe502a03c782483a17a5bce1e0900f31267730d670582c4fc56" }, "pipfile-spec": 6, "requires": { @@ -115,11 +115,11 @@ }, "pyspark": { "hashes": [ - "sha256:dd6569e547365eadc4f887bf57f153e4d582a68c4b490de475d55b9981664910" + "sha256:6effc9ce98edf231f4d683fd14f7270629bf8458c628d6a2620ded4bb34f3cb9" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==3.5.1" + "version": "==3.5.5" }, "sparkdataframecomparer": { "hashes": [ diff --git a/VERSION b/VERSION index 013adb7..db7a480 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.30 +0.1.31