From 37c68b8517417a69ce60dee57f45ee7ae7ee9883 Mon Sep 17 00:00:00 2001 From: marcus Date: Wed, 8 Jul 2026 14:29:48 -0500 Subject: [PATCH 1/2] [SPARK-57964][PYSPARK] Fix DataFrame.dropna VALUE_NOT_ANY_OR_ALL parameter name mismatch --- python/pyspark/sql/classic/dataframe.py | 2 +- python/pyspark/sql/tests/test_stat.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python/pyspark/sql/classic/dataframe.py b/python/pyspark/sql/classic/dataframe.py index a4288270a5ad4..dfe67a64aa53a 100644 --- a/python/pyspark/sql/classic/dataframe.py +++ b/python/pyspark/sql/classic/dataframe.py @@ -1356,7 +1356,7 @@ def dropna( if how is not None and how not in ["any", "all"]: raise PySparkValueError( errorClass="VALUE_NOT_ANY_OR_ALL", - messageParameters={"arg_name": "how", "arg_type": how}, + messageParameters={"arg_name": "how", "arg_value": how}, ) if subset is None: diff --git a/python/pyspark/sql/tests/test_stat.py b/python/pyspark/sql/tests/test_stat.py index e3655e24c968d..441198f190f2d 100644 --- a/python/pyspark/sql/tests/test_stat.py +++ b/python/pyspark/sql/tests/test_stat.py @@ -28,6 +28,7 @@ from pyspark.errors import ( AnalysisException, PySparkTypeError, + PySparkValueError, ) from pyspark.testing.sqlutils import ReusedSQLTestCase @@ -127,6 +128,16 @@ def test_dropna(self): }, ) + # Regression test: invalid 'how' should raise PySparkValueError, not AssertionError + with self.assertRaises(PySparkValueError) as pe: + self.spark.createDataFrame([("Alice", 50, 80.1)], schema).dropna(how="foo") + + self.check_error( + exception=pe.exception, + errorClass="VALUE_NOT_ANY_OR_ALL", + messageParameters={"arg_name": "how", "arg_value": "foo"}, + ) + def test_fillna(self): schema = StructType( [ From a9358608685a6359ce8be7fdf771ad2328bbe43f Mon Sep 17 00:00:00 2001 From: marcus Date: Wed, 8 Jul 2026 16:18:58 -0500 Subject: [PATCH 2/2] Fix Connect dropna to use VALUE_NOT_ANY_OR_ALL (consistent with classic path) --- python/pyspark/sql/connect/dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py index 7551210036c9e..6ee47341639c5 100644 --- a/python/pyspark/sql/connect/dataframe.py +++ b/python/pyspark/sql/connect/dataframe.py @@ -1394,7 +1394,7 @@ def dropna( min_non_nulls = None else: raise PySparkValueError( - errorClass="CANNOT_BE_EMPTY", + errorClass="VALUE_NOT_ANY_OR_ALL", messageParameters={"arg_name": "how", "arg_value": str(how)}, )