From 79f7da2aca334dd5bdf3e4d43540eafb69d649f3 Mon Sep 17 00:00:00 2001 From: Avi Shinnar Date: Thu, 3 Sep 2026 14:44:47 +0000 Subject: [PATCH 1/3] run more CI tests with python 3.14 Signed-off-by: Avi Shinnar --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc4a27a2c..d2293a8ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -129,7 +129,7 @@ jobs: - test/test_autogen_lib.py - test/test_relational.py - test/test_category_encoders.py - python-version: ['3.13'] + python-version: ['3.14'] setup-target: ['.[full,test]'] include: - test-case: test/test_core_misc.py @@ -221,7 +221,7 @@ jobs: - test/test_pipeline.py - test/test_relational.py - test/test_category_encoders.py - python-version: ['3.11', '3.12', '3.13'] + python-version: ['3.11', '3.12', '3.13', '3.14'] setup-target: ['.[full,test]'] include: - test-case: test/test_core_transformers.py From 59246f188a2c036de7dd8e51b5137dc759bcebd9 Mon Sep 17 00:00:00 2001 From: Avi Shinnar Date: Thu, 10 Sep 2026 15:04:30 +0000 Subject: [PATCH 2/3] remove version bound on pyspark Signed-off-by: Avi Shinnar --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3c6f2ed5a..2573ed141 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ test = [ "sphinxcontrib.apidoc", "sphinxcontrib-svg2pdfconverter", "pytest", - "pyspark<4.0.0", + "pyspark", "func_timeout", "category-encoders", "pynisher==0.6.4", From 5eea16ee56583bd70deedf997830cfe2cdd697ce Mon Sep 17 00:00:00 2001 From: Avi Shinnar Date: Fri, 11 Sep 2026 00:57:56 +0000 Subject: [PATCH 3/3] pyspark fixes Signed-off-by: Avi Shinnar --- lale/datasets/data_schemas.py | 22 ++++++++++++++++++++-- lale/lib/rasl/ordinal_encoder.py | 2 ++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lale/datasets/data_schemas.py b/lale/datasets/data_schemas.py index f0597d7fc..aa352f41f 100644 --- a/lale/datasets/data_schemas.py +++ b/lale/datasets/data_schemas.py @@ -46,10 +46,18 @@ from pyspark.sql import DataFrame as SparkDataFrame from pyspark.sql import GroupedData as SparkGroupedData + try: + # PySpark 4.x: concrete DataFrame lives in pyspark.sql.classic.dataframe + from pyspark.sql.classic.dataframe import DataFrame as _SparkDataFrameBase + except ImportError: + # PySpark 3.x: pyspark.sql.DataFrame is already the concrete class + _SparkDataFrameBase = SparkDataFrame # type: ignore[assignment] + except ImportError: Py4JError = None SparkDataFrame = None SparkGroupedData = None + _SparkDataFrameBase = None # type: ignore[assignment] # See instructions for subclassing numpy ndarray: @@ -115,7 +123,12 @@ def _gen_index_name(df, cpt=None): else: return name - class SparkDataFrameWithIndex(SparkDataFrame): # type: ignore + class SparkDataFrameWithIndex(_SparkDataFrameBase): # type: ignore + def __new__(cls, *args, **kwargs): + # Override __new__ to bypass pyspark.sql.DataFrame.__new__ which + # requires (jdf, sql_ctx) positional args (changed in PySpark 4.x). + return object.__new__(cls) + def __init__(self, df, index_names=None): if index_names is not None and len(index_names) == 1: index_name = index_names[0] @@ -135,7 +148,12 @@ def __init__(self, df, index_names=None): table_name = get_table_name(df) if table_name is not None: df_with_index = df_with_index.alias(table_name) - super().__init__(df_with_index._jdf, df_with_index.sql_ctx) + # PySpark 4.x deprecated sql_ctx in favour of sparkSession; + # keep sql_ctx as fallback for older PySpark versions. + session = ( + getattr(df_with_index, "sparkSession", None) or df_with_index.sql_ctx + ) + super().__init__(df_with_index._jdf, session) self.index_name = index_name self.index_names = index_names for f in df.schema.fieldNames(): diff --git a/lale/lib/rasl/ordinal_encoder.py b/lale/lib/rasl/ordinal_encoder.py index c1b28fbd6..f3575fe8d 100644 --- a/lale/lib/rasl/ordinal_encoder.py +++ b/lale/lib/rasl/ordinal_encoder.py @@ -100,6 +100,8 @@ def simplify_val(v): return int(v) if safe_issubdtype(type(v), np.floating): return float(v) + if isinstance(v, np.str_): + return str(v) return v result = Map(