[SPARK-58037] [PYTHON] [TESTS] Add DataFrame API golden file test framework for PySpark

dev/.rat-excludes

@@ -141,6 +141,13 @@ ui-test/package-lock.json
 core/src/main/resources/org/apache/spark/ui/static/package.json
 testCommitLog
 .*\.har
+.*\.test
 spark-logo.svg
 spark-logo-rev.svg
 .nojekyll
+# PySpark DataFrame golden test fixture scripts
+# (python/pyspark/sql/tests/df_golden/scripts/<group>/). These are executable
+# test fixtures, not source, so they are excluded like the SQL golden inputs
+# (see .*\.sql above). Add one directory name per golden group below. RAT matches
+# path components, so each name must be unique across the whole repository.
+group_by

dev/sparktestsupport/modules.py

@@ -637,6 +637,8 @@ def __hash__(self):
         "pyspark.sql.tests.coercion.test_python_udf_input_type",
         "pyspark.sql.tests.coercion.test_pandas_udf_return_type",
         "pyspark.sql.tests.coercion.test_python_udf_return_type",
+        "pyspark.sql.tests.df_golden.test_df_golden",
+        "pyspark.sql.tests.df_golden.test_df_golden_framework",
     ],
 )
 

pyproject.toml

@@ -59,6 +59,9 @@ ignore = [
     "python/pyspark/errors/error_classes.py" = ["E501"]
     # Examples contain some unused variables.
     "examples/src/main/python/sql/datasource.py" = ["F841"]
+    # DataFrame golden test scripts are executed by the framework with ``spark``
+    # injected into their namespace, so it is undefined at lint time.
+    "python/pyspark/sql/tests/df_golden/scripts/**/*.py" = ["F821"]
 
 [tool.black]
 # When changing the version, we have to update

python/MANIFEST.in

@@ -17,6 +17,7 @@
 # Reference: https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html
 
 recursive-include pyspark *.pyi py.typed *.json
+recursive-include pyspark/sql/tests/df_golden *.test *.py
 recursive-include deps/jars *.jar
 graft deps/bin
 recursive-include deps/sbin spark-config.sh spark-daemon.sh start-history-server.sh stop-history-server.sh

python/pyspark/sql/tests/df_golden/__init__.py

@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#