From e4d80aff891f6a3cb17a29b733bcbbc23e7527c7 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:40:34 +0530 Subject: [PATCH 1/6] Skip ORC timezone tests that need subprocesses --- python/pyarrow/tests/test_orc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pyarrow/tests/test_orc.py b/python/pyarrow/tests/test_orc.py index b5b38006e5a7..5ad4e059ed25 100644 --- a/python/pyarrow/tests/test_orc.py +++ b/python/pyarrow/tests/test_orc.py @@ -145,6 +145,7 @@ def test_example_using_json(filename, datadir): check_example_file(path, table, need_fix=True) +@pytest.mark.processes def test_timezone_database_absent(datadir): # Example file relies on the timezone "US/Pacific". It should gracefully # fail, not crash, if the timezone database is not found. @@ -165,6 +166,7 @@ def test_timezone_database_absent(datadir): subprocess.run([sys.executable, "-c", code], check=True) +@pytest.mark.processes def test_timezone_absent(datadir, tmpdir): # Example file relies on the timezone "US/Pacific". It should gracefully # fail, not crash, if the timezone database is present but the timezone From 3f93705cb9407d48778e8971c49c3c8017fc7ea1 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:41:23 +0530 Subject: [PATCH 2/6] Remove Node driver --- python/scripts/run_emscripten_tests.py | 75 +------------------------- 1 file changed, 1 insertion(+), 74 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 4e555bb1f5d2..bbdc6443411f 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -21,8 +21,6 @@ import http.server import os import queue -import shutil -import subprocess import sys import time import threading @@ -144,75 +142,6 @@ def launch_server(dist_dir): p.terminate() -class NodeDriver: - import subprocess - - def __init__(self, hostname, port): - # handle different syntax for util-linux script and macOS BSD-style script - if sys.platform == "darwin": - script_command = [shutil.which("script"), "-q", "/dev/null", - shutil.which("node")] - else: - script_command = [shutil.which("script"), "-c", shutil.which("node")] - self.process = subprocess.Popen( - script_command, - stdin=subprocess.PIPE, - shell=False, - bufsize=0, - ) - print(self.process) - time.sleep(0.1) # wait for node to start - self.hostname = hostname - self.port = port - self.last_ret_code = None - - def load_pyodide(self, dist_dir): - self.execute_js( - f""" - const {{ loadPyodide }} = require('{dist_dir}/pyodide.js'); - let pyodide = await loadPyodide(); - """ - ) - - def clear_logs(self): - pass # we don't handle logs for node - - def write_stdin(self, buffer): - # because we use unbuffered IO for - # stdout, stdin.write is also unbuffered - # so might under-run on writes - while len(buffer) > 0 and self.process.poll() is None: - written = self.process.stdin.write(buffer) - if written == len(buffer): - break - elif written == 0: - # full buffer - wait - time.sleep(0.01) - else: - buffer = buffer[written:] - - def execute_js(self, code, wait_for_terminate=True): - self.write_stdin((code + "\n").encode("utf-8")) - - def load_arrow(self): - self.execute_js(f"await pyodide.loadPackage('{PYARROW_WHEEL_PATH}');") - - def execute_python(self, code, wait_for_terminate=True): - js_code = f""" - python = `{code}`; - await pyodide.loadPackagesFromImports(python); - python_output = await pyodide.runPythonAsync(python); - """ - self.last_ret_code = self.execute_js(js_code, wait_for_terminate) - return self.last_ret_code - - def wait_for_done(self): - # in node we just let it run above - # then send EOF and join process - self.write_stdin(b"process.exit(python_output)\n") - return self.process.wait() - - class BrowserDriver: def __init__(self, hostname, port, driver): self.url = f"http://{hostname}:{port}/test.html" @@ -363,7 +292,7 @@ def _importorskip(modname, minversion=None, reason=None, *, "-r", "--runtime", type=str, - choices=["chrome", "node", "firefox"], + choices=["chrome", "firefox"], help="Runtime to run tests in", default="chrome", ) @@ -376,8 +305,6 @@ def _importorskip(modname, minversion=None, reason=None, *, with launch_server(dist_dir) as (hostname, port): if args.runtime == "chrome": driver = ChromeDriver(hostname, port) - elif args.runtime == "node": - driver = NodeDriver(hostname, port) elif args.runtime == "firefox": driver = FirefoxDriver(hostname, port) From 01ac7d55880719fea647f04bc488b8bf868de72b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:44:02 +0530 Subject: [PATCH 3/6] Run Emscripten Node tests in `pyodide venv` --- ci/scripts/python_test_emscripten.sh | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/ci/scripts/python_test_emscripten.sh b/ci/scripts/python_test_emscripten.sh index 621ac4bd0d18..f267b2abb6db 100755 --- a/ci/scripts/python_test_emscripten.sh +++ b/ci/scripts/python_test_emscripten.sh @@ -24,6 +24,10 @@ set -ex build_dir=${1}/python pyodide_dist_dir=${2} +# keep the virtual environment outside of the source tree, so that +# pytest imports the installed wheel and does not pick up the +# python/setup.cfg (its faulthandler_timeout setting needs threads) +pyodide_venv_dir=${1}/pyodide-venv cd "${build_dir}" @@ -32,7 +36,28 @@ cd "${build_dir}" pyodide_wheel=$(ls -t dist/pyarrow*.whl | head -1) echo "-------------- Running emscripten tests in Node ----------------------" -python scripts/run_emscripten_tests.py "${pyodide_wheel}" --dist-dir="${pyodide_dist_dir}" --runtime=node +# A Pyodide virtual environment runs Python in Node.js, and +# its pip installs pyemscripten wheels +rm -rf "${pyodide_venv_dir}" +pyodide venv "${pyodide_venv_dir}" +"${pyodide_venv_dir}/bin/pip" install \ + "${pyodide_wheel}" \ + cffi \ + cloudpickle \ + fsspec \ + hypothesis \ + packaging \ + pandas \ + pytest \ + pytz \ + scipy \ + tzdata + +pushd "${pyodide_venv_dir}" +# FIXME: Substrait expression serialization crashes the interpreter +# with a "Cannot convert a BigInt value to a number" error. +"${pyodide_venv_dir}/bin/python" -m pytest -r s -m "not substrait" --pyargs pyarrow +popd echo "-------------- Running emscripten tests in Chrome --------------------" python scripts/run_emscripten_tests.py "${pyodide_wheel}" --dist-dir="${pyodide_dist_dir}" --runtime=chrome From 10cae88fe7ea28f7f28156fa2307eef03387d209 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:44:22 +0530 Subject: [PATCH 4/6] Add optional dependencies to browser runner too --- python/scripts/run_emscripten_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index bbdc6443411f..fa36164f376a 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -259,6 +259,10 @@ def _load_pyarrow_in_runner(driver, wheel_name): await pjs.loadPackage("tzdata") await pjs.loadPackage("numpy") await pjs.loadPackage("pandas") + await pjs.loadPackage("cffi") + await pjs.loadPackage("cloudpickle") + await pjs.loadPackage("fsspec") + await pjs.loadPackage("scipy") import pytest import inspect if "exc_type" not in inspect.signature(pytest.importorskip).parameters: From 5dce9f4aff070bca9fc21a20f5561668cabbe1cc Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:58:16 +0530 Subject: [PATCH 5/6] Substrait tests don't seem to fail any long --- ci/scripts/python_test_emscripten.sh | 4 +--- python/scripts/run_emscripten_tests.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/ci/scripts/python_test_emscripten.sh b/ci/scripts/python_test_emscripten.sh index f267b2abb6db..935331dd21f0 100755 --- a/ci/scripts/python_test_emscripten.sh +++ b/ci/scripts/python_test_emscripten.sh @@ -54,9 +54,7 @@ pyodide venv "${pyodide_venv_dir}" tzdata pushd "${pyodide_venv_dir}" -# FIXME: Substrait expression serialization crashes the interpreter -# with a "Cannot convert a BigInt value to a number" error. -"${pyodide_venv_dir}/bin/python" -m pytest -r s -m "not substrait" --pyargs pyarrow +"${pyodide_venv_dir}/bin/python" -m pytest -r s --pyargs pyarrow popd echo "-------------- Running emscripten tests in Chrome --------------------" diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index fa36164f376a..0eedb8442769 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -322,9 +322,7 @@ def _importorskip(modname, minversion=None, reason=None, *, """ import pyarrow,pathlib pyarrow_dir = pathlib.Path(pyarrow.__file__).parent -# Substrait expression serialization crashes pyodide with a -# "Cannot convert a BigInt value to a number" error. -pytest.main([pyarrow_dir, '-r', 's', '-m', 'not substrait']) +pytest.main([pyarrow_dir, '-r', 's']) """, wait_for_terminate=False, ) From 470540ea27d525466323af5a3ba095733eb69638 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:58:29 +0530 Subject: [PATCH 6/6] Add docs about testing PyArrow pyemscripten wheel --- docs/source/developers/cpp/emscripten.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/source/developers/cpp/emscripten.rst b/docs/source/developers/cpp/emscripten.rst index f8af9b990978..90aba0a315f8 100644 --- a/docs/source/developers/cpp/emscripten.rst +++ b/docs/source/developers/cpp/emscripten.rst @@ -75,6 +75,23 @@ go to ``arrow/python`` and run It should make a wheel targeting the currently enabled version of Pyodide in the ``dist`` subdirectory. +To test the wheel, you can create a Pyodide virtual environment, which +runs Python in Node.js, and run the PyArrow test suite in it: + +.. code:: shell + + # create the environment outside of the source tree, so that the + # installed wheel is imported and python/setup.cfg is not used + pyodide venv /tmp/pyarrow-pyodide-venv + source /tmp/pyarrow-pyodide-venv/bin/activate + pip install dist/pyarrow-*.whl hypothesis pandas pytest + cd /tmp/pyarrow-pyodide-venv + python -m pytest --pyargs pyarrow + +The tests can also be run in a browser with +``python/scripts/run_emscripten_tests.py``, which needs Selenium and a +`Pyodide distribution `_. + Manual Build ------------