diff --git a/compose.yaml b/compose.yaml index dead873fee5e..f253caaaf514 100644 --- a/compose.yaml +++ b/compose.yaml @@ -982,7 +982,7 @@ services: clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} pyodide_version: "0.26.0" - chrome_version: "148" + chrome_version: "latest" selenium_version: "4.41.0" required_python_min: "(3,12)" python: ${PYTHON} diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 07054b4bb295..26b3e37a496a 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2073,7 +2073,10 @@ function(build_protobuf) "-DCMAKE_C_FLAGS=" "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_HOST_PREFIX}" -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_DEBUG_POSTFIX=) + -Dprotobuf_DEBUG_POSTFIX= + # OFF so protobuf does not take conda's zlib and add its include dir + # globally, causing vendored Abseil sources to include env's old headers + -Dprotobuf_WITH_ZLIB=OFF) if(ABSL_VENDORED) # Force protobuf to reuse Arrow's already-extracted absl source # so we don't re-download and we don't have issues with multiple abseil. diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 3cd7d5fd67b3..99e0e42ed9c1 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -82,22 +82,36 @@ def do_GET(self) -> bytes | None: elif self.path.endswith("/worker.js"): body = b""" importScripts("./pyodide.js"); + function do_print(arg) { + let databytes = Array.from(arg); + self.postMessage({print:databytes}); + return databytes.length; + } + function log(message) { + do_print(new TextEncoder().encode(`[worker] ${message}\n`)); + } onmessage = async function (e) { const data = e.data; - if (!self.pyodide) { - self.pyodide = await loadPyodide(); - } - function do_print(arg) { - let databytes = Array.from(arg); - self.postMessage({print:databytes}); - return databytes.length; + try { + if (!self.pyodide) { + log("loading Pyodide"); + self.pyodide = await loadPyodide(); + log("loaded Pyodide"); + } + self.pyodide.setStdout({write:do_print,isatty:data.isatty}); + self.pyodide.setStderr({write:do_print,isatty:data.isatty}); + + log("loading packages from imports"); + await self.pyodide.loadPackagesFromImports(data.python); + log("running Python"); + let results = await self.pyodide.runPythonAsync(data.python); + log("Python completed"); + self.postMessage({results}); + } catch (error) { + do_print(new TextEncoder().encode( + `Worker failed: ${error && error.stack || error}\n`)); + self.postMessage({results: 1}); } - self.pyodide.setStdout({write:do_print,isatty:data.isatty}); - self.pyodide.setStderr({write:do_print,isatty:data.isatty}); - - await self.pyodide.loadPackagesFromImports(data.python); - let results = await self.pyodide.runPythonAsync(data.python); - self.postMessage({results}); } """ self.send_response(200) @@ -215,30 +229,23 @@ def load_arrow(self): ) def execute_python(self, code, wait_for_terminate=True): + self.driver.execute_script( + f""" + let python = `{code}`; + delete window.python_script_done; + window.python_done_callback= (x) => {{window.python_script_done=x;}}; + window.pyworker.postMessage( + {{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}}); + """ + ) if wait_for_terminate: - self.driver.execute_async_script( - f""" - let callback = arguments[arguments.length-1]; - python = `{code}`; - window.python_done_callback = callback; - window.pyworker.postMessage( - {{python, isatty: {'true' if sys.stdout.isatty() else 'false'}}}); - """ - ) - else: - self.driver.execute_script( - f""" - let python = `{code}`; - window.python_done_callback= (x) => {{window.python_script_done=x;}}; - window.pyworker.postMessage( - {{python,isatty:{'true' if sys.stdout.isatty() else 'false'}}}); - """ - ) + return self.wait_for_done() def clear_logs(self): self.driver.execute_script("window.python_logs = [];") def wait_for_done(self): + last_heartbeat = time.monotonic() while True: # poll for console.log messages from our webworker # which are the output of pytest @@ -247,11 +254,16 @@ def wait_for_done(self): ) if len(lines) > 0: sys.stdout.buffer.write(bytes(lines)) + sys.stdout.buffer.flush() done = self.driver.execute_script("return window.python_script_done;") if done is not None: value = done["result"] self.driver.execute_script("delete window.python_script_done;") return value + now = time.monotonic() + if now - last_heartbeat > 30: + print("Waiting for browser-side Python to finish...", flush=True) + last_heartbeat = now time.sleep(0.1) @@ -289,6 +301,14 @@ def _load_pyarrow_in_runner(driver, wheel_name): await pjs.loadPackage("numpy") await pjs.loadPackage("pandas") import pytest + import inspect + if "exc_type" not in inspect.signature(pytest.importorskip).parameters: + _original_importorskip = pytest.importorskip + def _importorskip(modname, minversion=None, reason=None, *, + exc_type=ImportError): + return _original_importorskip(modname, minversion=minversion, + reason=reason) + pytest.importorskip = _importorskip import pandas # import pandas after pyarrow package load for pandas/pyarrow # functions to work import pyarrow