diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile index e5fb9ff238c8..b7ae4d23e180 100644 --- a/ci/docker/conda-python-emscripten.dockerfile +++ b/ci/docker/conda-python-emscripten.dockerfile @@ -18,32 +18,39 @@ ARG repo ARG arch ARG arch_short -ARG python="3.12" +ARG python="3.14" FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG selenium_version="4.41.0" -ARG pyodide_version="0.27.1" +ARG pyodide_version="314.0.7" +# pyodide-build is versioned independently of Pyodide +ARG pyodide_build_version="0.39.2" ARG chrome_version="latest" -ARG required_python_min="(3,12)" -# fail if python version < 3.12 +ARG required_python_min="(3,14)" +# fail if python version < 3.14 RUN echo "check PYTHON>=${required_python_min}" && python -c "import sys;sys.exit(0 if sys.version_info>=${required_python_min} else 1)" -# install selenium and recent pyodide-build and recent python +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + libatomic1 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # needs to be a login shell so ~/.profile is read SHELL ["/bin/bash", "--login", "-c", "-o", "pipefail"] +# install selenium and recent pyodide-build and recent python RUN python -m pip install --no-cache-dir selenium==${selenium_version} && \ - python -m pip install --no-cache-dir --upgrade pyodide-build>=${pyodide_version} + python -m pip install --no-cache-dir --upgrade "pyodide-build>=${pyodide_build_version}" # install pyodide dist directory to /pyodide RUN pyodide_dist_url="https://github.com/pyodide/pyodide/releases/download/${pyodide_version}/pyodide-${pyodide_version}.tar.bz2" && \ wget -q "${pyodide_dist_url}" -O- | tar -xj -C / -# install node 20 (needed for async call support) +# install node 24 (needed for async call support and JSPI) # and pthread-stubs for build, and unzip needed for chrome build to work # xz is needed by emsdk to extract node tarballs -RUN conda install nodejs=20 unzip pthread-stubs make xz -c conda-forge +RUN conda install nodejs=24 unzip pthread-stubs make xz -c conda-forge # install correct version of emscripten for this pyodide COPY ci/scripts/install_emscripten.sh /arrow/ci/scripts/ diff --git a/compose.yaml b/compose.yaml index a3a65c4c2d04..a7831dbd9f93 100644 --- a/compose.yaml +++ b/compose.yaml @@ -982,10 +982,11 @@ services: arch_short: ${ARCH_SHORT} clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} - pyodide_version: "0.27.1" + pyodide_version: "314.0.7" + pyodide_build_version: "0.39.2" chrome_version: "latest" selenium_version: "4.41.0" - required_python_min: "(3,12)" + required_python_min: "(3,14)" python: ${PYTHON} shm_size: *shm-size volumes: *ubuntu-volumes diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 750e3bef29c0..66973f0e0eb8 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -714,26 +714,35 @@ if(MSVC) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - # flags are: + # For the Pyodide 2026_0 ABI, the flags are: # 1) We force *everything* to build as position independent - # 2) And with support for C++ exceptions - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fexceptions") + # 2) We support C++ WASM exceptions, using the native WebAssembly + # exception handling proposal and WebAssembly-based setjmp/longjmp + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fwasm-exceptions -sSUPPORT_LONGJMP=wasm") # deprecated-literal-operator error is thrown in datetime (vendored lib in arrow) set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -fPIC -fexceptions -Wno-error=deprecated-literal-operator") + "${CMAKE_CXX_FLAGS} -fPIC -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -Wno-error=deprecated-literal-operator" + ) # flags for creating shared libraries (only used in pyarrow, because # Emscripten builds libarrow as static) # flags are: # 1) Tell it to use JavaScript / WebAssembly 64 bit number support. - # 2) Tell it to build with support for C++ exceptions + # 2) Tell it to build with support for C++ exceptions (see above) # 3) Skip linker flags error which happens with -soname parameter - set(ARROW_EMSCRIPTEN_LINKER_FLAGS "-sWASM_BIGINT=1 -fexceptions -Wno-error=linkflags") - set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS - "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}") - set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS - "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}") + # 4) Record $ORIGIN as an RPATH, so that Pyodide can locate + # libarrow_python.so next to the extension modules needing it + # Note: the CMAKE_SHARED_LIBRARY_CREATE__FLAGS and + # CMAKE_SHARED_MODULE_CREATE__FLAGS variables are not used by the + # Ninja generator, so the flags must go through the linker-flags + # variables, which apply to SHARED (libarrow_python) and MODULE (Cython + # extension) targets respectively. + set(ARROW_EMSCRIPTEN_LINKER_FLAGS + "-sWASM_BIGINT=1 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -Wno-error=linkflags") + set(ARROW_EMSCRIPTEN_DYLIB_CREATE_FLAGS + "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS} '-Wl,-rpath,$ORIGIN'") + set(CMAKE_SHARED_LINKER_FLAGS "${ARROW_EMSCRIPTEN_DYLIB_CREATE_FLAGS}") + set(CMAKE_MODULE_LINKER_FLAGS "${ARROW_EMSCRIPTEN_DYLIB_CREATE_FLAGS}") if(ARROW_TESTING) # Deeply nested tests need a larger stack (1 MiB). set(ARROW_EMSCRIPTEN_TEST_STACK_SIZE 1048576) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 123def5b66c5..ec106c36f5e0 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2171,6 +2171,10 @@ function(build_protobuf) set(PROTOBUF_HOST_CMAKE_ARGS "-DCMAKE_CXX_FLAGS=" "-DCMAKE_C_FLAGS=" + # Explicitly set the C++ standard for the host build. Without this, + # Abseil may be compiled with the host compiler's default standard, + # which can be older than C++17. + "-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}" "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_HOST_PREFIX}" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_DEBUG_POSTFIX= diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d9351d3afe0a..c3ddbc91f9a0 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -493,7 +493,7 @@ tasks: params: env: UBUNTU: 22.04 - PYTHON: 3.12 + PYTHON: 3.14 image: conda-python-emscripten test-conda-python-3.11-hypothesis: diff --git a/docs/source/developers/cpp/emscripten.rst b/docs/source/developers/cpp/emscripten.rst index d41b69aa6f0a..f8af9b990978 100644 --- a/docs/source/developers/cpp/emscripten.rst +++ b/docs/source/developers/cpp/emscripten.rst @@ -34,9 +34,9 @@ activate it using the commands below (see https://emscripten.org/docs/getting_st git clone https://github.com/emscripten-core/emsdk.git cd emsdk # replace with the desired EMSDK version. - # e.g. for Pyodide 0.26, you need EMSDK version 3.1.58 + # e.g. for Pyodide 314.0.x, you need EMSDK version 5.0.3 # the versions can be found in the Makefile.envs file in the Pyodide repo: - # https://github.com/pyodide/pyodide/blob/10b484cfe427e076c929a55dc35cfff01ea8d3bc/Makefile.envs + # https://github.com/pyodide/pyodide/blob/314.0.7/Makefile.envs ./emsdk install ./emsdk activate source ./emsdk_env.sh @@ -49,8 +49,9 @@ versions of emsdk tools. .. code:: shell # install Pyodide build tools. - # e.g., for version 0.26 of Pyodide, pyodide-build 0.26 and later work - pip install "pyodide-build>=0.26" + # pyodide-build is versioned independently of Pyodide. + # For Pyodide 314.0.x, pyodide-build 0.39.2 and later work well + pip install "pyodide-build>=0.39.2" Then build with the ``ninja-release-emscripten`` CMake preset, like below: diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 5bff8bcc6499..750f0f06f576 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -111,7 +111,9 @@ if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1") set(CMAKE_EXPORT_COMPILE_COMMANDS 1) endif() -if(UNIX) +# On Emscripten, the equivalent $ORIGIN runtime path is added +# through the linker flags in SetupCxxFlags.cmake instead +if(UNIX AND NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) # In the event that we are bundling the shared libraries (e.g. in a # manylinux1 wheel), we need to set the RPATH of the extensions to the diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index 275a7e808ddd..79e651de6b4d 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -2202,6 +2202,7 @@ def test_write_csv_decimal(tmpdir, type_factory): assert out.column('col').cast(type) == table.column('col') +@pytest.mark.threading def test_read_csv_gil_deadlock(): # GH-38676 # This test depends on several preconditions: diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 9ac3e0d8d7f8..4e555bb1f5d2 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -76,7 +76,7 @@ def do_GET(self) -> bytes | None: evt.data.print.forEach((x)=>{window.python_logs.push(x)}); } } - window.pyworker = new Worker("worker.js"); + window.pyworker = new Worker("worker.js", {type: "module"}); window.pyworker.onmessage = capturelogs; @@ -90,8 +90,8 @@ def do_GET(self) -> bytes | None: self.copyfile(BytesIO(body), self.wfile) elif self.path.endswith("/worker.js"): body = b""" - importScripts("./pyodide.js"); - onmessage = async function (e) { + import { loadPyodide } from "./pyodide.mjs"; + self.onmessage = async function (e) { const data = e.data; if (!self.pyodide) { self.pyodide = await loadPyodide(); @@ -148,8 +148,14 @@ class NodeDriver: import subprocess def __init__(self, hostname, port): + # handle different syntax for util-linux script and macOS BSD-style script + if sys.platform == "darwin": + script_command = [shutil.which("script"), "-q", "/dev/null", + shutil.which("node")] + else: + script_command = [shutil.which("script"), "-c", shutil.which("node")] self.process = subprocess.Popen( - [shutil.which("script"), "-c", shutil.which("node")], + script_command, stdin=subprocess.PIPE, shell=False, bufsize=0, @@ -320,6 +326,7 @@ def _load_pyarrow_in_runner(driver, wheel_name): if "pyarrow" not in sys.modules: await micropip.install("hypothesis") import pyodide_js as pjs + await pjs.loadPackage("packaging") await pjs.loadPackage("tzdata") await pjs.loadPackage("numpy") await pjs.loadPackage("pandas")