From fb5d8e5b90dbe3a3e0b5c2b00bd183a345eaeaa3 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 8 Jul 2026 18:11:12 +0200 Subject: [PATCH 1/6] Verify fix --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index dead873fee5e..f253caaaf514 100644 --- a/compose.yaml +++ b/compose.yaml @@ -982,7 +982,7 @@ services: clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} pyodide_version: "0.26.0" - chrome_version: "148" + chrome_version: "latest" selenium_version: "4.41.0" required_python_min: "(3,12)" python: ${PYTHON} From 95de4e1f4edd8c4b9efeb67bd7fe65a8ffdbdf76 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 8 Jul 2026 22:08:35 +0200 Subject: [PATCH 2/6] Disable zlib --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 07054b4bb295..703368e92a60 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2073,7 +2073,10 @@ function(build_protobuf) "-DCMAKE_C_FLAGS=" "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_HOST_PREFIX}" -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_DEBUG_POSTFIX=) + -Dprotobuf_DEBUG_POSTFIX= + # OFF so protobuf does not take conda's zlib and add its include dir + # globally, so vendored Abseil sources compile against env's old headers + -Dprotobuf_WITH_ZLIB=OFF) if(ABSL_VENDORED) # Force protobuf to reuse Arrow's already-extracted absl source # so we don't re-download and we don't have issues with multiple abseil. From c3346c0cd9de0a02e08a8dcd39c8ab0d8026cbb2 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Wed, 8 Jul 2026 22:54:19 +0200 Subject: [PATCH 3/6] Install newer pytest veersion --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- python/scripts/run_emscripten_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 703368e92a60..26b3e37a496a 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2075,7 +2075,7 @@ function(build_protobuf) -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_DEBUG_POSTFIX= # OFF so protobuf does not take conda's zlib and add its include dir - # globally, so vendored Abseil sources compile against env's old headers + # globally, causing vendored Abseil sources to include env's old headers -Dprotobuf_WITH_ZLIB=OFF) if(ABSL_VENDORED) # Force protobuf to reuse Arrow's already-extracted absl source diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 3cd7d5fd67b3..05557a4c465f 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -283,7 +283,7 @@ def _load_pyarrow_in_runner(driver, wheel_name): """import sys import micropip if "pyarrow" not in sys.modules: - await micropip.install("hypothesis") + await micropip.install(["hypothesis", "pytest>=8.2"]) import pyodide_js as pjs await pjs.loadPackage("tzdata") await pjs.loadPackage("numpy") From 010098204ade73196833249f5b9b70060d7c5dd4 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 07:54:06 +0200 Subject: [PATCH 4/6] Keep pytest8.1.1 plus patch for exec_type --- python/scripts/run_emscripten_tests.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 05557a4c465f..002698a77e2b 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -283,12 +283,20 @@ def _load_pyarrow_in_runner(driver, wheel_name): """import sys import micropip if "pyarrow" not in sys.modules: - await micropip.install(["hypothesis", "pytest>=8.2"]) + await micropip.install("hypothesis") import pyodide_js as pjs await pjs.loadPackage("tzdata") await pjs.loadPackage("numpy") await pjs.loadPackage("pandas") import pytest + import inspect + if "exc_type" not in inspect.signature(pytest.importorskip).parameters: + _original_importorskip = pytest.importorskip + def _importorskip(modname, minversion=None, reason=None, *, + exc_type=ImportError): + return _original_importorskip(modname, minversion=minversion, + reason=reason) + pytest.importorskip = _importorskip import pandas # import pandas after pyarrow package load for pandas/pyarrow # functions to work import pyarrow From e00ae746eb5ff95d337b2252b7135cef3493f593 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 10:13:20 +0200 Subject: [PATCH 5/6] Fix script timeout chrome 150 --- python/scripts/run_emscripten_tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 002698a77e2b..330b7d89cee1 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -33,6 +33,9 @@ from selenium import webdriver +BROWSER_TIMEOUT_SECONDS = 3600 + + class TemplateOverrider(http.server.SimpleHTTPRequestHandler): def log_request(self, code="-", size="-"): # don't log successful requests but log errors @@ -203,7 +206,7 @@ def __init__(self, hostname, port, driver): self.driver = driver self.driver.get(f"http://{hostname}:{port}/test.html") # Chrome on CI takes longer than locally to compile. - self.driver.set_script_timeout(1200) + self.driver.set_script_timeout(BROWSER_TIMEOUT_SECONDS) def load_pyodide(self, dist_dir): pass @@ -262,8 +265,9 @@ def __init__(self, hostname, port): options = Options() options.add_argument("--headless") options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") driver = webdriver.Chrome(options=options) - driver.command_executor._client_config.timeout = 1200 + driver.command_executor._client_config.timeout = BROWSER_TIMEOUT_SECONDS super().__init__(hostname, port, driver) From 5f42bde8460ffbbfa994c2e4fdee8e85db071113 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 9 Jul 2026 11:20:39 +0200 Subject: [PATCH 6/6] Add logging --- python/scripts/run_emscripten_tests.py | 74 +++++++++++++++----------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 330b7d89cee1..a4a57ad95688 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -85,22 +85,36 @@ def do_GET(self) -> bytes | None: elif self.path.endswith("/worker.js"): body = b""" importScripts("./pyodide.js"); + function postLog(message) { + self.postMessage({print: Array.from(new TextEncoder().encode( + `[worker] ${new Date().toISOString()} ${message}\n`))}); + } onmessage = async function (e) { const data = e.data; - if (!self.pyodide) { - self.pyodide = await loadPyodide(); - } - function do_print(arg) { - let databytes = Array.from(arg); - self.postMessage({print:databytes}); - return databytes.length; + try { + if (!self.pyodide) { + postLog("loading Pyodide"); + self.pyodide = await loadPyodide(); + postLog("loaded Pyodide"); + } + function do_print(arg) { + let databytes = Array.from(arg); + self.postMessage({print:databytes}); + return databytes.length; + } + self.pyodide.setStdout({write:do_print,isatty:data.isatty}); + self.pyodide.setStderr({write:do_print,isatty:data.isatty}); + + postLog("loading packages from imports"); + await self.pyodide.loadPackagesFromImports(data.python); + postLog("running Python"); + let results = await self.pyodide.runPythonAsync(data.python); + postLog(`Python completed with result ${results}`); + self.postMessage({results}); + } catch (error) { + postLog(`Python failed: ${error && error.stack || error}`); + self.postMessage({results: 1}); } - self.pyodide.setStdout({write:do_print,isatty:data.isatty}); - self.pyodide.setStderr({write:do_print,isatty:data.isatty}); - - await self.pyodide.loadPackagesFromImports(data.python); - let results = await self.pyodide.runPythonAsync(data.python); - self.postMessage({results}); } """ self.send_response(200) @@ -218,30 +232,23 @@ def load_arrow(self): ) def execute_python(self, code, wait_for_terminate=True): + self.driver.execute_script( + f""" + let python = `{code}`; + delete window.python_script_done; + window.python_done_callback= (x) => {{window.python_script_done=x;}}; + window.pyworker.postMessage( + {{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}}); + """ + ) if wait_for_terminate: - self.driver.execute_async_script( - f""" - let callback = arguments[arguments.length-1]; - python = `{code}`; - window.python_done_callback = callback; - window.pyworker.postMessage( - {{python, isatty: {'true' if sys.stdout.isatty() else 'false'}}}); - """ - ) - else: - self.driver.execute_script( - f""" - let python = `{code}`; - window.python_done_callback= (x) => {{window.python_script_done=x;}}; - window.pyworker.postMessage( - {{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}}); - """ - ) + return self.wait_for_done() def clear_logs(self): self.driver.execute_script("window.python_logs = [];") def wait_for_done(self): + last_heartbeat = time.monotonic() while True: # poll for console.log messages from our webworker # which are the output of pytest @@ -250,11 +257,16 @@ def wait_for_done(self): ) if len(lines) > 0: sys.stdout.buffer.write(bytes(lines)) + sys.stdout.buffer.flush() done = self.driver.execute_script("return window.python_script_done;") if done is not None: value = done["result"] self.driver.execute_script("delete window.python_script_done;") return value + now = time.monotonic() + if now - last_heartbeat > 30: + print("Waiting for browser-side Python to finish...", flush=True) + last_heartbeat = now time.sleep(0.1)