From ed1331d082dc98c8ec2fd51f9ffc7fecb3ab92ac Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 14 Aug 2025 02:17:22 +0530 Subject: [PATCH 01/26] Bump to Pyodide 0.28.1 --- ci/docker/conda-python-emscripten.dockerfile | 12 ++++++------ compose.yaml | 4 ++-- docs/source/developers/cpp/emscripten.rst | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile index e5fb9ff238c8..1d1f5f4beffd 100644 --- a/ci/docker/conda-python-emscripten.dockerfile +++ b/ci/docker/conda-python-emscripten.dockerfile @@ -18,14 +18,14 @@ ARG repo ARG arch ARG arch_short -ARG python="3.12" +ARG python="3.13" FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG selenium_version="4.41.0" -ARG pyodide_version="0.27.1" +ARG pyodide_version="0.28.1" ARG chrome_version="latest" -ARG required_python_min="(3,12)" -# fail if python version < 3.12 +ARG required_python_min="(3,13)" +# fail if python version < 3.13 RUN echo "check PYTHON>=${required_python_min}" && python -c "import sys;sys.exit(0 if sys.version_info>=${required_python_min} else 1)" # install selenium and recent pyodide-build and recent python @@ -40,10 +40,10 @@ RUN python -m pip install --no-cache-dir selenium==${selenium_version} && \ RUN pyodide_dist_url="https://github.com/pyodide/pyodide/releases/download/${pyodide_version}/pyodide-${pyodide_version}.tar.bz2" && \ wget -q "${pyodide_dist_url}" -O- | tar -xj -C / -# install node 20 (needed for async call support) +# install node 22 (needed for async call support and JSPI) # and pthread-stubs for build, and unzip needed for chrome build to work # xz is needed by emsdk to extract node tarballs -RUN conda install nodejs=20 unzip pthread-stubs make xz -c conda-forge +RUN conda install nodejs=22 unzip pthread-stubs make xz -c conda-forge # install correct version of emscripten for this pyodide COPY ci/scripts/install_emscripten.sh /arrow/ci/scripts/ diff --git a/compose.yaml b/compose.yaml index a3a65c4c2d04..aad43654331e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -982,10 +982,10 @@ services: arch_short: ${ARCH_SHORT} clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} - pyodide_version: "0.27.1" + pyodide_version: "0.28.1" chrome_version: "latest" selenium_version: "4.41.0" - required_python_min: "(3,12)" + required_python_min: "(3,13)" python: ${PYTHON} shm_size: *shm-size volumes: *ubuntu-volumes diff --git a/docs/source/developers/cpp/emscripten.rst b/docs/source/developers/cpp/emscripten.rst index d41b69aa6f0a..785c9b0a67c5 100644 --- a/docs/source/developers/cpp/emscripten.rst +++ b/docs/source/developers/cpp/emscripten.rst @@ -34,9 +34,9 @@ activate it using the commands below (see https://emscripten.org/docs/getting_st git clone https://github.com/emscripten-core/emsdk.git cd emsdk # replace with the desired EMSDK version. - # e.g. for Pyodide 0.26, you need EMSDK version 3.1.58 + # e.g. for Pyodide 0.28, you need EMSDK version 4.0.9 # the versions can be found in the Makefile.envs file in the Pyodide repo: - # https://github.com/pyodide/pyodide/blob/10b484cfe427e076c929a55dc35cfff01ea8d3bc/Makefile.envs + # https://github.com/pyodide/pyodide/blob/0db5ff79f310694a7d72ae0b3279e9809ab836d6/Makefile.envs ./emsdk install ./emsdk activate source ./emsdk_env.sh From 6899a0bf69e3b1e4bf9e021204d9b3cdf6767168 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 14 Aug 2025 09:03:54 +0530 Subject: [PATCH 02/26] Bump Python version for conda-python-emscripten --- dev/tasks/tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d9351d3afe0a..78aff6f0d99d 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -493,7 +493,7 @@ tasks: params: env: UBUNTU: 22.04 - PYTHON: 3.12 + PYTHON: 3.13 image: conda-python-emscripten test-conda-python-3.11-hypothesis: From 18b10d3586ba73b3872292b2d6f338e33e5f8e4d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 14 Aug 2025 09:29:33 +0530 Subject: [PATCH 03/26] Pass the right flag for WASM exception handling --- cpp/cmake_modules/SetupCxxFlags.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 750e3bef29c0..0a7450af9a36 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -717,18 +717,18 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") # flags are: # 1) We force *everything* to build as position independent # 2) And with support for C++ exceptions - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fexceptions") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fwasm-exceptions") # deprecated-literal-operator error is thrown in datetime (vendored lib in arrow) set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -fPIC -fexceptions -Wno-error=deprecated-literal-operator") + "${CMAKE_CXX_FLAGS} -fPIC -fwasm-exceptions -Wno-error=deprecated-literal-operator") # flags for creating shared libraries (only used in pyarrow, because # Emscripten builds libarrow as static) # flags are: # 1) Tell it to use JavaScript / WebAssembly 64 bit number support. - # 2) Tell it to build with support for C++ exceptions + # 2) Tell it to build with support for WASM exceptions # 3) Skip linker flags error which happens with -soname parameter - set(ARROW_EMSCRIPTEN_LINKER_FLAGS "-sWASM_BIGINT=1 -fexceptions -Wno-error=linkflags") + set(ARROW_EMSCRIPTEN_LINKER_FLAGS "-sWASM_BIGINT=1 -fwasm-exceptions -Wno-error=linkflags") set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}") set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS From dc2cc6fe23a5c73145544068078286b84f9e9674 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 14 Aug 2025 10:48:50 +0530 Subject: [PATCH 04/26] Install `libatomic1` for Pyodide Dockerfile --- ci/docker/conda-python-emscripten.dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile index 1d1f5f4beffd..18e3a9d33aed 100644 --- a/ci/docker/conda-python-emscripten.dockerfile +++ b/ci/docker/conda-python-emscripten.dockerfile @@ -28,11 +28,14 @@ ARG required_python_min="(3,13)" # fail if python version < 3.13 RUN echo "check PYTHON>=${required_python_min}" && python -c "import sys;sys.exit(0 if sys.version_info>=${required_python_min} else 1)" -# install selenium and recent pyodide-build and recent python +RUN apt-get update -y -q && \ + apt install -y -q --no-install-recommends \ + libatomic1 # needs to be a login shell so ~/.profile is read SHELL ["/bin/bash", "--login", "-c", "-o", "pipefail"] +# install selenium and recent pyodide-build and recent python RUN python -m pip install --no-cache-dir selenium==${selenium_version} && \ python -m pip install --no-cache-dir --upgrade pyodide-build>=${pyodide_version} From 71592709492342ce360a6e6e3c68b7b0522ea791 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:49:27 +0530 Subject: [PATCH 05/26] Undo WASM EH flags as SjLj is enabled This reverts commit eebb7a86201a1c5fa7110ce172ab78c479fdbc02. --- cpp/cmake_modules/SetupCxxFlags.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 0a7450af9a36..750e3bef29c0 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -717,18 +717,18 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") # flags are: # 1) We force *everything* to build as position independent # 2) And with support for C++ exceptions - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fwasm-exceptions") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fexceptions") # deprecated-literal-operator error is thrown in datetime (vendored lib in arrow) set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -fPIC -fwasm-exceptions -Wno-error=deprecated-literal-operator") + "${CMAKE_CXX_FLAGS} -fPIC -fexceptions -Wno-error=deprecated-literal-operator") # flags for creating shared libraries (only used in pyarrow, because # Emscripten builds libarrow as static) # flags are: # 1) Tell it to use JavaScript / WebAssembly 64 bit number support. - # 2) Tell it to build with support for WASM exceptions + # 2) Tell it to build with support for C++ exceptions # 3) Skip linker flags error which happens with -soname parameter - set(ARROW_EMSCRIPTEN_LINKER_FLAGS "-sWASM_BIGINT=1 -fwasm-exceptions -Wno-error=linkflags") + set(ARROW_EMSCRIPTEN_LINKER_FLAGS "-sWASM_BIGINT=1 -fexceptions -Wno-error=linkflags") set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}") set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS From 79c32fc8eae96375142755db1fd5eb91859d8cea Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:01:19 +0530 Subject: [PATCH 06/26] Fix lint in Emscripten Dockerfile `apt-get` command --- ci/docker/conda-python-emscripten.dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile index 18e3a9d33aed..7bfe7f47c572 100644 --- a/ci/docker/conda-python-emscripten.dockerfile +++ b/ci/docker/conda-python-emscripten.dockerfile @@ -29,8 +29,10 @@ ARG required_python_min="(3,13)" RUN echo "check PYTHON>=${required_python_min}" && python -c "import sys;sys.exit(0 if sys.version_info>=${required_python_min} else 1)" RUN apt-get update -y -q && \ - apt install -y -q --no-install-recommends \ - libatomic1 + apt-get install -y -q --no-install-recommends \ + libatomic1 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # needs to be a login shell so ~/.profile is read SHELL ["/bin/bash", "--login", "-c", "-o", "pipefail"] From 2cb09bcd9082aec135b2a9131051b88fc1f1fcf5 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 14 Aug 2025 13:10:14 +0530 Subject: [PATCH 07/26] Build static `libarrow_python` on Emscripten --- python/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 5bff8bcc6499..9bab598ee7fc 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -514,7 +514,11 @@ if(NOT PYARROW_CPP_LINK_LIBS) endif() endif() -add_library(arrow_python SHARED ${PYARROW_CPP_SRCS}) +if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + add_library(arrow_python STATIC ${PYARROW_CPP_SRCS}) +else() + add_library(arrow_python SHARED ${PYARROW_CPP_SRCS}) +endif() target_include_directories(arrow_python PUBLIC ${PYARROW_CPP_ROOT_DIR} ${CMAKE_CURRENT_BINARY_DIR}/pyarrow/src) From 36411af74649fb7fd261523d52dd569078fc3851 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 15 Aug 2025 01:36:01 +0530 Subject: [PATCH 08/26] Add Emscripten-specific flags for `_acero` --- python/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 9bab598ee7fc..04d4f7ba8d9e 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -803,7 +803,11 @@ endif() # Acero if(PYARROW_BUILD_ACERO) - if(ARROW_BUILD_SHARED) + if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + set(ACERO_LINK_LIBS) + set_property(TARGET _acero PROPERTY LINK_FLAGS + "-s SIDE_MODULE=1 -s WASM_BIGINT -s EXPORT_ALL=1") + elseif(ARROW_BUILD_SHARED) if(PYARROW_BUNDLE_ARROW_CPP) bundle_arrow_lib(${ARROW_ACERO_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION}) if(MSVC) From 7891c6c44603a5a5d0848e1436ffe1473c812133 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 15 Aug 2025 02:00:20 +0530 Subject: [PATCH 09/26] Fix CMake lint --- python/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 04d4f7ba8d9e..a0bdc72960b6 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -805,8 +805,8 @@ endif() if(PYARROW_BUILD_ACERO) if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set(ACERO_LINK_LIBS) - set_property(TARGET _acero PROPERTY LINK_FLAGS - "-s SIDE_MODULE=1 -s WASM_BIGINT -s EXPORT_ALL=1") + set_property(TARGET _acero PROPERTY LINK_FLAGS + "-s SIDE_MODULE=1 -s WASM_BIGINT -s EXPORT_ALL=1") elseif(ARROW_BUILD_SHARED) if(PYARROW_BUNDLE_ARROW_CPP) bundle_arrow_lib(${ARROW_ACERO_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION}) From e3350598bb24b29c7f8bf6b700c3d848afbbdfb1 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 15 Aug 2025 04:36:33 +0530 Subject: [PATCH 10/26] Revert "Fix CMake lint" This reverts commit f73c495170f88bd529a541252f76a3cb4e300fdf. --- python/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index a0bdc72960b6..04d4f7ba8d9e 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -805,8 +805,8 @@ endif() if(PYARROW_BUILD_ACERO) if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set(ACERO_LINK_LIBS) - set_property(TARGET _acero PROPERTY LINK_FLAGS - "-s SIDE_MODULE=1 -s WASM_BIGINT -s EXPORT_ALL=1") + set_property(TARGET _acero PROPERTY LINK_FLAGS + "-s SIDE_MODULE=1 -s WASM_BIGINT -s EXPORT_ALL=1") elseif(ARROW_BUILD_SHARED) if(PYARROW_BUNDLE_ARROW_CPP) bundle_arrow_lib(${ARROW_ACERO_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION}) From b01d943aedc5719e586bc75b023e6fd0fc9e8fd7 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 15 Aug 2025 04:36:42 +0530 Subject: [PATCH 11/26] Revert "Add Emscripten-specific flags for `_acero`" This reverts commit 781b54a5a7c65fd7dd5cf6c919c561c312c762dc. --- python/CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 04d4f7ba8d9e..9bab598ee7fc 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -803,11 +803,7 @@ endif() # Acero if(PYARROW_BUILD_ACERO) - if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - set(ACERO_LINK_LIBS) - set_property(TARGET _acero PROPERTY LINK_FLAGS - "-s SIDE_MODULE=1 -s WASM_BIGINT -s EXPORT_ALL=1") - elseif(ARROW_BUILD_SHARED) + if(ARROW_BUILD_SHARED) if(PYARROW_BUNDLE_ARROW_CPP) bundle_arrow_lib(${ARROW_ACERO_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION}) if(MSVC) From ce9030310123e7923c09517dc63573c0ac191b9a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 15 Aug 2025 06:33:01 +0530 Subject: [PATCH 12/26] Revert "Build static `libarrow_python` on Emscripten" This reverts commit 20de7ad163a606840765c229ae95e9c40422ca72. --- python/CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 9bab598ee7fc..5bff8bcc6499 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -514,11 +514,7 @@ if(NOT PYARROW_CPP_LINK_LIBS) endif() endif() -if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - add_library(arrow_python STATIC ${PYARROW_CPP_SRCS}) -else() - add_library(arrow_python SHARED ${PYARROW_CPP_SRCS}) -endif() +add_library(arrow_python SHARED ${PYARROW_CPP_SRCS}) target_include_directories(arrow_python PUBLIC ${PYARROW_CPP_ROOT_DIR} ${CMAKE_CURRENT_BINARY_DIR}/pyarrow/src) From ab1754ca21f0b0789d103d6f82d1e9c719d0ec4e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:14:53 +0530 Subject: [PATCH 13/26] Update to Python 3.14 and Pyodide 314.0.4 --- ci/docker/conda-python-emscripten.dockerfile | 8 ++++---- compose.yaml | 4 ++-- dev/tasks/tasks.yml | 2 +- docs/source/developers/cpp/emscripten.rst | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile index 7bfe7f47c572..bfa2a9056623 100644 --- a/ci/docker/conda-python-emscripten.dockerfile +++ b/ci/docker/conda-python-emscripten.dockerfile @@ -18,14 +18,14 @@ ARG repo ARG arch ARG arch_short -ARG python="3.13" +ARG python="3.14" FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG selenium_version="4.41.0" -ARG pyodide_version="0.28.1" +ARG pyodide_version="314.0.4" ARG chrome_version="latest" -ARG required_python_min="(3,13)" -# fail if python version < 3.13 +ARG required_python_min="(3,14)" +# fail if python version < 3.14 RUN echo "check PYTHON>=${required_python_min}" && python -c "import sys;sys.exit(0 if sys.version_info>=${required_python_min} else 1)" RUN apt-get update -y -q && \ diff --git a/compose.yaml b/compose.yaml index aad43654331e..9ee830930405 100644 --- a/compose.yaml +++ b/compose.yaml @@ -982,10 +982,10 @@ services: arch_short: ${ARCH_SHORT} clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} - pyodide_version: "0.28.1" + pyodide_version: "314.0.4" chrome_version: "latest" selenium_version: "4.41.0" - required_python_min: "(3,13)" + required_python_min: "(3,14)" python: ${PYTHON} shm_size: *shm-size volumes: *ubuntu-volumes diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 78aff6f0d99d..c3ddbc91f9a0 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -493,7 +493,7 @@ tasks: params: env: UBUNTU: 22.04 - PYTHON: 3.13 + PYTHON: 3.14 image: conda-python-emscripten test-conda-python-3.11-hypothesis: diff --git a/docs/source/developers/cpp/emscripten.rst b/docs/source/developers/cpp/emscripten.rst index 785c9b0a67c5..7f284be63ce3 100644 --- a/docs/source/developers/cpp/emscripten.rst +++ b/docs/source/developers/cpp/emscripten.rst @@ -34,9 +34,9 @@ activate it using the commands below (see https://emscripten.org/docs/getting_st git clone https://github.com/emscripten-core/emsdk.git cd emsdk # replace with the desired EMSDK version. - # e.g. for Pyodide 0.28, you need EMSDK version 4.0.9 + # e.g. for Pyodide 314.0.x, you need EMSDK version 5.0.3 # the versions can be found in the Makefile.envs file in the Pyodide repo: - # https://github.com/pyodide/pyodide/blob/0db5ff79f310694a7d72ae0b3279e9809ab836d6/Makefile.envs + # https://github.com/pyodide/pyodide/blob/314.0.4/Makefile.envs ./emsdk install ./emsdk activate source ./emsdk_env.sh From 5798061d80e1a941781a68792cf07aa3f432bbcf Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:24:17 +0530 Subject: [PATCH 14/26] Use Pyodide 2026_0 ABI flags and set `RPATH` to `$ORIGIN` --- cpp/cmake_modules/SetupCxxFlags.cmake | 31 +++++++++++++++++---------- python/CMakeLists.txt | 4 +++- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 750e3bef29c0..66973f0e0eb8 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -714,26 +714,35 @@ if(MSVC) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - # flags are: + # For the Pyodide 2026_0 ABI, the flags are: # 1) We force *everything* to build as position independent - # 2) And with support for C++ exceptions - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fexceptions") + # 2) We support C++ WASM exceptions, using the native WebAssembly + # exception handling proposal and WebAssembly-based setjmp/longjmp + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fwasm-exceptions -sSUPPORT_LONGJMP=wasm") # deprecated-literal-operator error is thrown in datetime (vendored lib in arrow) set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} -fPIC -fexceptions -Wno-error=deprecated-literal-operator") + "${CMAKE_CXX_FLAGS} -fPIC -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -Wno-error=deprecated-literal-operator" + ) # flags for creating shared libraries (only used in pyarrow, because # Emscripten builds libarrow as static) # flags are: # 1) Tell it to use JavaScript / WebAssembly 64 bit number support. - # 2) Tell it to build with support for C++ exceptions + # 2) Tell it to build with support for C++ exceptions (see above) # 3) Skip linker flags error which happens with -soname parameter - set(ARROW_EMSCRIPTEN_LINKER_FLAGS "-sWASM_BIGINT=1 -fexceptions -Wno-error=linkflags") - set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS - "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}") - set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS - "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}") + # 4) Record $ORIGIN as an RPATH, so that Pyodide can locate + # libarrow_python.so next to the extension modules needing it + # Note: the CMAKE_SHARED_LIBRARY_CREATE__FLAGS and + # CMAKE_SHARED_MODULE_CREATE__FLAGS variables are not used by the + # Ninja generator, so the flags must go through the linker-flags + # variables, which apply to SHARED (libarrow_python) and MODULE (Cython + # extension) targets respectively. + set(ARROW_EMSCRIPTEN_LINKER_FLAGS + "-sWASM_BIGINT=1 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -Wno-error=linkflags") + set(ARROW_EMSCRIPTEN_DYLIB_CREATE_FLAGS + "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS} '-Wl,-rpath,$ORIGIN'") + set(CMAKE_SHARED_LINKER_FLAGS "${ARROW_EMSCRIPTEN_DYLIB_CREATE_FLAGS}") + set(CMAKE_MODULE_LINKER_FLAGS "${ARROW_EMSCRIPTEN_DYLIB_CREATE_FLAGS}") if(ARROW_TESTING) # Deeply nested tests need a larger stack (1 MiB). set(ARROW_EMSCRIPTEN_TEST_STACK_SIZE 1048576) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 5bff8bcc6499..750f0f06f576 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -111,7 +111,9 @@ if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1") set(CMAKE_EXPORT_COMPILE_COMMANDS 1) endif() -if(UNIX) +# On Emscripten, the equivalent $ORIGIN runtime path is added +# through the linker flags in SetupCxxFlags.cmake instead +if(UNIX AND NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) # In the event that we are bundling the shared libraries (e.g. in a # manylinux1 wheel), we need to set the RPATH of the extensions to the From 83eed5d081dbdb26636a5ae58973739bede06bc1 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:26:36 +0530 Subject: [PATCH 15/26] `packaging` needed on Emscripten due to `pyarrow/tests/util.py` --- python/scripts/run_emscripten_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 9ac3e0d8d7f8..be7e05429fbe 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -320,6 +320,7 @@ def _load_pyarrow_in_runner(driver, wheel_name): if "pyarrow" not in sys.modules: await micropip.install("hypothesis") import pyodide_js as pjs + await pjs.loadPackage("packaging") await pjs.loadPackage("tzdata") await pjs.loadPackage("numpy") await pjs.loadPackage("pandas") From 8309f9ffff1a00f055cdbeb5939d1cce2abe8cdb Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:29:38 +0530 Subject: [PATCH 16/26] Add handling for Unix `script -c`/`script -q` --- python/scripts/run_emscripten_tests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index be7e05429fbe..57ea4814b7ba 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -148,8 +148,14 @@ class NodeDriver: import subprocess def __init__(self, hostname, port): + # handle different syntax for util-linux script and macOS BSD-style script + if sys.platform == "darwin": + script_command = [shutil.which("script"), "-q", "/dev/null", + shutil.which("node")] + else: + script_command = [shutil.which("script"), "-c", shutil.which("node")] self.process = subprocess.Popen( - [shutil.which("script"), "-c", shutil.which("node")], + script_command, stdin=subprocess.PIPE, shell=False, bufsize=0, From 20b54fb0032d4b08cc3c1bdd18101d61c81cb731 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:55:16 +0530 Subject: [PATCH 17/26] Fix Abseil ABI mismatch on linking with AppleClang --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 123def5b66c5..ec106c36f5e0 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2171,6 +2171,10 @@ function(build_protobuf) set(PROTOBUF_HOST_CMAKE_ARGS "-DCMAKE_CXX_FLAGS=" "-DCMAKE_C_FLAGS=" + # Explicitly set the C++ standard for the host build. Without this, + # Abseil may be compiled with the host compiler's default standard, + # which can be older than C++17. + "-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}" "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_HOST_PREFIX}" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_DEBUG_POSTFIX= From 03bddf505e11d12a2c70b97569e4e1abd4428add Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:14:54 +0530 Subject: [PATCH 18/26] Update to Node 24 in environment Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> --- ci/docker/conda-python-emscripten.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile index bfa2a9056623..b02fc52663fd 100644 --- a/ci/docker/conda-python-emscripten.dockerfile +++ b/ci/docker/conda-python-emscripten.dockerfile @@ -45,10 +45,10 @@ RUN python -m pip install --no-cache-dir selenium==${selenium_version} && \ RUN pyodide_dist_url="https://github.com/pyodide/pyodide/releases/download/${pyodide_version}/pyodide-${pyodide_version}.tar.bz2" && \ wget -q "${pyodide_dist_url}" -O- | tar -xj -C / -# install node 22 (needed for async call support and JSPI) +# install node 24 (needed for async call support and JSPI) # and pthread-stubs for build, and unzip needed for chrome build to work # xz is needed by emsdk to extract node tarballs -RUN conda install nodejs=22 unzip pthread-stubs make xz -c conda-forge +RUN conda install nodejs=24 unzip pthread-stubs make xz -c conda-forge # install correct version of emscripten for this pyodide COPY ci/scripts/install_emscripten.sh /arrow/ci/scripts/ From 49021dd40c65b675d2f6b6c5592c59c4abc23e2c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:29:50 +0530 Subject: [PATCH 19/26] Add separate pyodide-build version argument --- ci/docker/conda-python-emscripten.dockerfile | 4 +++- compose.yaml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile index b02fc52663fd..5848734d2d98 100644 --- a/ci/docker/conda-python-emscripten.dockerfile +++ b/ci/docker/conda-python-emscripten.dockerfile @@ -23,6 +23,8 @@ FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG selenium_version="4.41.0" ARG pyodide_version="314.0.4" +# pyodide-build is versioned independently of Pyodide +ARG pyodide_build_version="0.39.0" ARG chrome_version="latest" ARG required_python_min="(3,14)" # fail if python version < 3.14 @@ -39,7 +41,7 @@ SHELL ["/bin/bash", "--login", "-c", "-o", "pipefail"] # install selenium and recent pyodide-build and recent python RUN python -m pip install --no-cache-dir selenium==${selenium_version} && \ - python -m pip install --no-cache-dir --upgrade pyodide-build>=${pyodide_version} + python -m pip install --no-cache-dir --upgrade "pyodide-build>=${pyodide_build_version}" # install pyodide dist directory to /pyodide RUN pyodide_dist_url="https://github.com/pyodide/pyodide/releases/download/${pyodide_version}/pyodide-${pyodide_version}.tar.bz2" && \ diff --git a/compose.yaml b/compose.yaml index 9ee830930405..a0e6e9a9081a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -983,6 +983,7 @@ services: clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} pyodide_version: "314.0.4" + pyodide_build_version: "0.39.0" chrome_version: "latest" selenium_version: "4.41.0" required_python_min: "(3,14)" From 8538f281c6d134fc3837eceff9d9e15954c618fa Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:00:56 +0530 Subject: [PATCH 20/26] Apply suggestion from @agriyakhetarpal --- docs/source/developers/cpp/emscripten.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/developers/cpp/emscripten.rst b/docs/source/developers/cpp/emscripten.rst index 7f284be63ce3..1b39edbefee4 100644 --- a/docs/source/developers/cpp/emscripten.rst +++ b/docs/source/developers/cpp/emscripten.rst @@ -36,7 +36,7 @@ activate it using the commands below (see https://emscripten.org/docs/getting_st # replace with the desired EMSDK version. # e.g. for Pyodide 314.0.x, you need EMSDK version 5.0.3 # the versions can be found in the Makefile.envs file in the Pyodide repo: - # https://github.com/pyodide/pyodide/blob/314.0.4/Makefile.envs + # https://github.com/pyodide/pyodide/blob/314.07/Makefile.envs ./emsdk install ./emsdk activate source ./emsdk_env.sh From d22dc71de245f0bf8e4d93ba61aa10fa4e61efdd Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:01:18 +0530 Subject: [PATCH 21/26] Apply suggestion from @agriyakhetarpal --- docs/source/developers/cpp/emscripten.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/developers/cpp/emscripten.rst b/docs/source/developers/cpp/emscripten.rst index 1b39edbefee4..0413fd17e977 100644 --- a/docs/source/developers/cpp/emscripten.rst +++ b/docs/source/developers/cpp/emscripten.rst @@ -36,7 +36,7 @@ activate it using the commands below (see https://emscripten.org/docs/getting_st # replace with the desired EMSDK version. # e.g. for Pyodide 314.0.x, you need EMSDK version 5.0.3 # the versions can be found in the Makefile.envs file in the Pyodide repo: - # https://github.com/pyodide/pyodide/blob/314.07/Makefile.envs + # https://github.com/pyodide/pyodide/blob/314.0.7/Makefile.envs ./emsdk install ./emsdk activate source ./emsdk_env.sh From 07b8b8e1d4d9269a55daeea2da05803f866e25da Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:22:16 +0530 Subject: [PATCH 22/26] Bump to Pyodide 314.0.7, pyodide-build 0.39.2 --- ci/docker/conda-python-emscripten.dockerfile | 4 ++-- compose.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile index 5848734d2d98..b7ae4d23e180 100644 --- a/ci/docker/conda-python-emscripten.dockerfile +++ b/ci/docker/conda-python-emscripten.dockerfile @@ -22,9 +22,9 @@ ARG python="3.14" FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python} ARG selenium_version="4.41.0" -ARG pyodide_version="314.0.4" +ARG pyodide_version="314.0.7" # pyodide-build is versioned independently of Pyodide -ARG pyodide_build_version="0.39.0" +ARG pyodide_build_version="0.39.2" ARG chrome_version="latest" ARG required_python_min="(3,14)" # fail if python version < 3.14 diff --git a/compose.yaml b/compose.yaml index a0e6e9a9081a..a7831dbd9f93 100644 --- a/compose.yaml +++ b/compose.yaml @@ -982,8 +982,8 @@ services: arch_short: ${ARCH_SHORT} clang_tools: ${CLANG_TOOLS} llvm: ${LLVM} - pyodide_version: "314.0.4" - pyodide_build_version: "0.39.0" + pyodide_version: "314.0.7" + pyodide_build_version: "0.39.2" chrome_version: "latest" selenium_version: "4.41.0" required_python_min: "(3,14)" From 439bef670c17d8c2ed9c2e3d65d0615139e4efb9 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:22:46 +0530 Subject: [PATCH 23/26] Update pyodide-build version in Emscripten docs --- docs/source/developers/cpp/emscripten.rst | 5 +++-- testing | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/source/developers/cpp/emscripten.rst b/docs/source/developers/cpp/emscripten.rst index 0413fd17e977..f8af9b990978 100644 --- a/docs/source/developers/cpp/emscripten.rst +++ b/docs/source/developers/cpp/emscripten.rst @@ -49,8 +49,9 @@ versions of emsdk tools. .. code:: shell # install Pyodide build tools. - # e.g., for version 0.26 of Pyodide, pyodide-build 0.26 and later work - pip install "pyodide-build>=0.26" + # pyodide-build is versioned independently of Pyodide. + # For Pyodide 314.0.x, pyodide-build 0.39.2 and later work well + pip install "pyodide-build>=0.39.2" Then build with the ``ninja-release-emscripten`` CMake preset, like below: diff --git a/testing b/testing index 5a26d35785ff..9ff285c88565 160000 --- a/testing +++ b/testing @@ -1 +1 @@ -Subproject commit 5a26d35785ff8761e56d79eebff971f6f737e18c +Subproject commit 9ff285c88565f0f6abc855918c6a342e70e4909c From 48022a78dff6ae9be199c6d527763f5b6dc088a8 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:14:17 +0530 Subject: [PATCH 24/26] Use module web worker in Emscripten test runner --- python/scripts/run_emscripten_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/scripts/run_emscripten_tests.py b/python/scripts/run_emscripten_tests.py index 57ea4814b7ba..4e555bb1f5d2 100644 --- a/python/scripts/run_emscripten_tests.py +++ b/python/scripts/run_emscripten_tests.py @@ -76,7 +76,7 @@ def do_GET(self) -> bytes | None: evt.data.print.forEach((x)=>{window.python_logs.push(x)}); } } - window.pyworker = new Worker("worker.js"); + window.pyworker = new Worker("worker.js", {type: "module"}); window.pyworker.onmessage = capturelogs; @@ -90,8 +90,8 @@ def do_GET(self) -> bytes | None: self.copyfile(BytesIO(body), self.wfile) elif self.path.endswith("/worker.js"): body = b""" - importScripts("./pyodide.js"); - onmessage = async function (e) { + import { loadPyodide } from "./pyodide.mjs"; + self.onmessage = async function (e) { const data = e.data; if (!self.pyodide) { self.pyodide = await loadPyodide(); From 3c80ba45d6862e77144ff33d379a5df70cb321b3 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:16:54 +0530 Subject: [PATCH 25/26] `test_read_csv_gil_deadlock` is likely parallel? --- python/pyarrow/tests/test_csv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index 275a7e808ddd..79e651de6b4d 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -2202,6 +2202,7 @@ def test_write_csv_decimal(tmpdir, type_factory): assert out.column('col').cast(type) == table.column('col') +@pytest.mark.threading def test_read_csv_gil_deadlock(): # GH-38676 # This test depends on several preconditions: From 099c659e473a1bb987d0938959cc8f07afb0e5a5 Mon Sep 17 00:00:00 2001 From: Tadeja Kadunc Date: Thu, 12 Nov 2026 13:45:19 +0100 Subject: [PATCH 26/26] Restore testing submodule rev --- testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing b/testing index 9ff285c88565..5a26d35785ff 160000 --- a/testing +++ b/testing @@ -1 +1 @@ -Subproject commit 9ff285c88565f0f6abc855918c6a342e70e4909c +Subproject commit 5a26d35785ff8761e56d79eebff971f6f737e18c