run more CI tests with python 3.14

.github/workflows/build.yml

@@ -129,7 +129,7 @@ jobs:
         - test/test_autogen_lib.py
         - test/test_relational.py
         - test/test_category_encoders.py
-        python-version: ['3.13']
+        python-version: ['3.14']
         setup-target: ['.[full,test]']
         include:
         - test-case: test/test_core_misc.py
@@ -221,7 +221,7 @@ jobs:
         - test/test_pipeline.py
         - test/test_relational.py
         - test/test_category_encoders.py
-        python-version: ['3.11', '3.12', '3.13']
+        python-version: ['3.11', '3.12', '3.13', '3.14']
         setup-target: ['.[full,test]']
         include:
         - test-case: test/test_core_transformers.py

lale/datasets/data_schemas.py

@@ -46,10 +46,18 @@
     from pyspark.sql import DataFrame as SparkDataFrame
     from pyspark.sql import GroupedData as SparkGroupedData
 
+    try:
+        # PySpark 4.x: concrete DataFrame lives in pyspark.sql.classic.dataframe
+        from pyspark.sql.classic.dataframe import DataFrame as _SparkDataFrameBase
+    except ImportError:
+        # PySpark 3.x: pyspark.sql.DataFrame is already the concrete class
+        _SparkDataFrameBase = SparkDataFrame  # type: ignore[assignment]
+
 except ImportError:
     Py4JError = None
     SparkDataFrame = None
     SparkGroupedData = None
+    _SparkDataFrameBase = None  # type: ignore[assignment]
 
 
 # See instructions for subclassing numpy ndarray:
@@ -115,7 +123,12 @@ def _gen_index_name(df, cpt=None):
         else:
             return name
 
-    class SparkDataFrameWithIndex(SparkDataFrame):  # type: ignore
+    class SparkDataFrameWithIndex(_SparkDataFrameBase):  # type: ignore
+        def __new__(cls, *args, **kwargs):
+            # Override __new__ to bypass pyspark.sql.DataFrame.__new__ which
+            # requires (jdf, sql_ctx) positional args (changed in PySpark 4.x).
+            return object.__new__(cls)
+
         def __init__(self, df, index_names=None):
             if index_names is not None and len(index_names) == 1:
                 index_name = index_names[0]
@@ -135,7 +148,12 @@ def __init__(self, df, index_names=None):
             table_name = get_table_name(df)
             if table_name is not None:
                 df_with_index = df_with_index.alias(table_name)
-            super().__init__(df_with_index._jdf, df_with_index.sql_ctx)
+            # PySpark 4.x deprecated sql_ctx in favour of sparkSession;
+            # keep sql_ctx as fallback for older PySpark versions.
+            session = (
+                getattr(df_with_index, "sparkSession", None) or df_with_index.sql_ctx
+            )
+            super().__init__(df_with_index._jdf, session)
             self.index_name = index_name
             self.index_names = index_names
             for f in df.schema.fieldNames():

lale/lib/rasl/ordinal_encoder.py

@@ -100,6 +100,8 @@ def simplify_val(v):
                 return int(v)
             if safe_issubdtype(type(v), np.floating):
                 return float(v)
+            if isinstance(v, np.str_):
+                return str(v)
             return v
 
         result = Map(

pyproject.toml

@@ -89,7 +89,7 @@ test = [
   "sphinxcontrib.apidoc",
   "sphinxcontrib-svg2pdfconverter",
   "pytest",
-  "pyspark<4.0.0",
+  "pyspark",
   "func_timeout",
   "category-encoders",
   "pynisher==0.6.4",