diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc4a27a2c..d2293a8ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -129,7 +129,7 @@ jobs: - test/test_autogen_lib.py - test/test_relational.py - test/test_category_encoders.py - python-version: ['3.13'] + python-version: ['3.14'] setup-target: ['.[full,test]'] include: - test-case: test/test_core_misc.py @@ -221,7 +221,7 @@ jobs: - test/test_pipeline.py - test/test_relational.py - test/test_category_encoders.py - python-version: ['3.11', '3.12', '3.13'] + python-version: ['3.11', '3.12', '3.13', '3.14'] setup-target: ['.[full,test]'] include: - test-case: test/test_core_transformers.py diff --git a/lale/datasets/data_schemas.py b/lale/datasets/data_schemas.py index f0597d7fc..aa352f41f 100644 --- a/lale/datasets/data_schemas.py +++ b/lale/datasets/data_schemas.py @@ -46,10 +46,18 @@ from pyspark.sql import DataFrame as SparkDataFrame from pyspark.sql import GroupedData as SparkGroupedData + try: + # PySpark 4.x: concrete DataFrame lives in pyspark.sql.classic.dataframe + from pyspark.sql.classic.dataframe import DataFrame as _SparkDataFrameBase + except ImportError: + # PySpark 3.x: pyspark.sql.DataFrame is already the concrete class + _SparkDataFrameBase = SparkDataFrame # type: ignore[assignment] + except ImportError: Py4JError = None SparkDataFrame = None SparkGroupedData = None + _SparkDataFrameBase = None # type: ignore[assignment] # See instructions for subclassing numpy ndarray: @@ -115,7 +123,12 @@ def _gen_index_name(df, cpt=None): else: return name - class SparkDataFrameWithIndex(SparkDataFrame): # type: ignore + class SparkDataFrameWithIndex(_SparkDataFrameBase): # type: ignore + def __new__(cls, *args, **kwargs): + # Override __new__ to bypass pyspark.sql.DataFrame.__new__ which + # requires (jdf, sql_ctx) positional args (changed in PySpark 4.x). + return object.__new__(cls) + def __init__(self, df, index_names=None): if index_names is not None and len(index_names) == 1: index_name = index_names[0] @@ -135,7 +148,12 @@ def __init__(self, df, index_names=None): table_name = get_table_name(df) if table_name is not None: df_with_index = df_with_index.alias(table_name) - super().__init__(df_with_index._jdf, df_with_index.sql_ctx) + # PySpark 4.x deprecated sql_ctx in favour of sparkSession; + # keep sql_ctx as fallback for older PySpark versions. + session = ( + getattr(df_with_index, "sparkSession", None) or df_with_index.sql_ctx + ) + super().__init__(df_with_index._jdf, session) self.index_name = index_name self.index_names = index_names for f in df.schema.fieldNames(): diff --git a/lale/lib/rasl/ordinal_encoder.py b/lale/lib/rasl/ordinal_encoder.py index c1b28fbd6..f3575fe8d 100644 --- a/lale/lib/rasl/ordinal_encoder.py +++ b/lale/lib/rasl/ordinal_encoder.py @@ -100,6 +100,8 @@ def simplify_val(v): return int(v) if safe_issubdtype(type(v), np.floating): return float(v) + if isinstance(v, np.str_): + return str(v) return v result = Map( diff --git a/pyproject.toml b/pyproject.toml index 3c6f2ed5a..2573ed141 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ test = [ "sphinxcontrib.apidoc", "sphinxcontrib-svg2pdfconverter", "pytest", - "pyspark<4.0.0", + "pyspark", "func_timeout", "category-encoders", "pynisher==0.6.4",