Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
ed1331d
Bump to Pyodide 0.28.1
agriyakhetarpal Aug 13, 2025
6899a0b
Bump Python version for conda-python-emscripten
agriyakhetarpal Aug 14, 2025
18b10d3
Pass the right flag for WASM exception handling
agriyakhetarpal Aug 14, 2025
dc2cc6f
Install `libatomic1` for Pyodide Dockerfile
agriyakhetarpal Aug 14, 2025
7159270
Undo WASM EH flags as SjLj is enabled
agriyakhetarpal Aug 14, 2025
79c32fc
Fix lint in Emscripten Dockerfile `apt-get` command
agriyakhetarpal Aug 14, 2025
2cb09bc
Build static `libarrow_python` on Emscripten
agriyakhetarpal Aug 14, 2025
36411af
Add Emscripten-specific flags for `_acero`
agriyakhetarpal Aug 14, 2025
7891c6c
Fix CMake lint
agriyakhetarpal Aug 14, 2025
e335059
Revert "Fix CMake lint"
agriyakhetarpal Aug 14, 2025
b01d943
Revert "Add Emscripten-specific flags for `_acero`"
agriyakhetarpal Aug 14, 2025
ce90303
Revert "Build static `libarrow_python` on Emscripten"
agriyakhetarpal Aug 15, 2025
ab1754c
Update to Python 3.14 and Pyodide 314.0.4
agriyakhetarpal Aug 15, 2026
5798061
Use Pyodide 2026_0 ABI flags and set `RPATH` to `$ORIGIN`
agriyakhetarpal Aug 15, 2026
83eed5d
`packaging` needed on Emscripten due to `pyarrow/tests/util.py`
agriyakhetarpal Aug 15, 2026
8309f9f
Add handling for Unix `script -c`/`script -q`
agriyakhetarpal Aug 15, 2026
20b54fb
Fix Abseil ABI mismatch on linking with AppleClang
agriyakhetarpal Aug 15, 2026
03bddf5
Update to Node 24 in environment
agriyakhetarpal Aug 15, 2026
49021dd
Add separate pyodide-build version argument
agriyakhetarpal Aug 15, 2026
8538f28
Apply suggestion from @agriyakhetarpal
agriyakhetarpal Oct 22, 2026
d22dc71
Apply suggestion from @agriyakhetarpal
agriyakhetarpal Oct 22, 2026
07b8b8e
Bump to Pyodide 314.0.7, pyodide-build 0.39.2
agriyakhetarpal Nov 12, 2026
439bef6
Update pyodide-build version in Emscripten docs
agriyakhetarpal Nov 12, 2026
48022a7
Use module web worker in Emscripten test runner
agriyakhetarpal Nov 12, 2026
3c80ba4
`test_read_csv_gil_deadlock` is likely parallel?
agriyakhetarpal Nov 12, 2026
099c659
Restore testing submodule rev
tadeja Nov 12, 2026
dafb6e2
Merge branch 'main' into feat/pyodide-0.28-builds
agriyakhetarpal Nov 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions ci/docker/conda-python-emscripten.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,32 +18,39 @@
ARG repo
ARG arch
ARG arch_short
ARG python="3.12"
ARG python="3.14"
FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python}

ARG selenium_version="4.41.0"
ARG pyodide_version="0.27.1"
ARG pyodide_version="314.0.7"
# pyodide-build is versioned independently of Pyodide
ARG pyodide_build_version="0.39.2"
ARG chrome_version="latest"
ARG required_python_min="(3,12)"
# fail if python version < 3.12
ARG required_python_min="(3,14)"
# fail if python version < 3.14
RUN echo "check PYTHON>=${required_python_min}" && python -c "import sys;sys.exit(0 if sys.version_info>=${required_python_min} else 1)"

# install selenium and recent pyodide-build and recent python
RUN apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
libatomic1 && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

# needs to be a login shell so ~/.profile is read
SHELL ["/bin/bash", "--login", "-c", "-o", "pipefail"]

# install selenium and recent pyodide-build and recent python
RUN python -m pip install --no-cache-dir selenium==${selenium_version} && \
python -m pip install --no-cache-dir --upgrade pyodide-build>=${pyodide_version}
python -m pip install --no-cache-dir --upgrade "pyodide-build>=${pyodide_build_version}"

# install pyodide dist directory to /pyodide
RUN pyodide_dist_url="https://github.com/pyodide/pyodide/releases/download/${pyodide_version}/pyodide-${pyodide_version}.tar.bz2" && \
wget -q "${pyodide_dist_url}" -O- | tar -xj -C /

# install node 20 (needed for async call support)
# install node 24 (needed for async call support and JSPI)
# and pthread-stubs for build, and unzip needed for chrome build to work
# xz is needed by emsdk to extract node tarballs
RUN conda install nodejs=20 unzip pthread-stubs make xz -c conda-forge
RUN conda install nodejs=24 unzip pthread-stubs make xz -c conda-forge

# install correct version of emscripten for this pyodide
COPY ci/scripts/install_emscripten.sh /arrow/ci/scripts/
Expand Down
5 changes: 3 additions & 2 deletions compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -982,10 +982,11 @@ services:
arch_short: ${ARCH_SHORT}
clang_tools: ${CLANG_TOOLS}
llvm: ${LLVM}
pyodide_version: "0.27.1"
pyodide_version: "314.0.7"
pyodide_build_version: "0.39.2"
chrome_version: "latest"
selenium_version: "4.41.0"
required_python_min: "(3,12)"
required_python_min: "(3,14)"
python: ${PYTHON}
shm_size: *shm-size
volumes: *ubuntu-volumes
Expand Down
31 changes: 20 additions & 11 deletions cpp/cmake_modules/SetupCxxFlags.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -714,26 +714,35 @@ if(MSVC)
endif()

if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
# flags are:
# For the Pyodide 2026_0 ABI, the flags are:
# 1) We force *everything* to build as position independent
# 2) And with support for C++ exceptions
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fexceptions")
# 2) We support C++ WASM exceptions, using the native WebAssembly
# exception handling proposal and WebAssembly-based setjmp/longjmp
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fwasm-exceptions -sSUPPORT_LONGJMP=wasm")
# deprecated-literal-operator error is thrown in datetime (vendored lib in arrow)
set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} -fPIC -fexceptions -Wno-error=deprecated-literal-operator")
"${CMAKE_CXX_FLAGS} -fPIC -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -Wno-error=deprecated-literal-operator"
)

# flags for creating shared libraries (only used in pyarrow, because
# Emscripten builds libarrow as static)
# flags are:
# 1) Tell it to use JavaScript / WebAssembly 64 bit number support.
# 2) Tell it to build with support for C++ exceptions
# 2) Tell it to build with support for C++ exceptions (see above)
# 3) Skip linker flags error which happens with -soname parameter
set(ARROW_EMSCRIPTEN_LINKER_FLAGS "-sWASM_BIGINT=1 -fexceptions -Wno-error=linkflags")
set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS
"-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}")
set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS
"-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}")
set(CMAKE_SHARED_LINKER_FLAGS "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}")
# 4) Record $ORIGIN as an RPATH, so that Pyodide can locate
# libarrow_python.so next to the extension modules needing it
# Note: the CMAKE_SHARED_LIBRARY_CREATE_<LANG>_FLAGS and
# CMAKE_SHARED_MODULE_CREATE_<LANG>_FLAGS variables are not used by the
# Ninja generator, so the flags must go through the linker-flags
# variables, which apply to SHARED (libarrow_python) and MODULE (Cython
# extension) targets respectively.
set(ARROW_EMSCRIPTEN_LINKER_FLAGS
"-sWASM_BIGINT=1 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -Wno-error=linkflags")
set(ARROW_EMSCRIPTEN_DYLIB_CREATE_FLAGS
"-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS} '-Wl,-rpath,$ORIGIN'")
set(CMAKE_SHARED_LINKER_FLAGS "${ARROW_EMSCRIPTEN_DYLIB_CREATE_FLAGS}")
set(CMAKE_MODULE_LINKER_FLAGS "${ARROW_EMSCRIPTEN_DYLIB_CREATE_FLAGS}")
if(ARROW_TESTING)
# Deeply nested tests need a larger stack (1 MiB).
set(ARROW_EMSCRIPTEN_TEST_STACK_SIZE 1048576)
Expand Down
4 changes: 4 additions & 0 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -2171,6 +2171,10 @@ function(build_protobuf)
set(PROTOBUF_HOST_CMAKE_ARGS
"-DCMAKE_CXX_FLAGS="
"-DCMAKE_C_FLAGS="
# Explicitly set the C++ standard for the host build. Without this,
# Abseil may be compiled with the host compiler's default standard,
# which can be older than C++17.
"-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}"
Comment on lines +2174 to +2177

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I hope this change is fine to keep here; this happened to me because I am using AppleClang 21 on my macOS machine to cross-compile for Emscripten.

The reason is that in the host protoc build, where CMAKE_CXX_FLAGS is blanked and no standard is set, absl compiles as C++14 and uses absl::string_view. That results in an ABI mismatch because Protobuf's targets reference the std::string_view signatures (since Protobuf compiles to C++17 or newer).

(I guess no one has hit this before, since the CI job exercises going from Linux to Emscripten, where gcc defaults to C++17)

"-DCMAKE_INSTALL_PREFIX=${PROTOBUF_HOST_PREFIX}"
-Dprotobuf_BUILD_TESTS=OFF
-Dprotobuf_DEBUG_POSTFIX=
Expand Down
2 changes: 1 addition & 1 deletion dev/tasks/tasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -493,7 +493,7 @@ tasks:
params:
env:
UBUNTU: 22.04
PYTHON: 3.12
PYTHON: 3.14
image: conda-python-emscripten

test-conda-python-3.11-hypothesis:
Expand Down
9 changes: 5 additions & 4 deletions docs/source/developers/cpp/emscripten.rst
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ activate it using the commands below (see https://emscripten.org/docs/getting_st
git clone https://github.com/emscripten-core/emsdk.git
cd emsdk
# replace <version> with the desired EMSDK version.
# e.g. for Pyodide 0.26, you need EMSDK version 3.1.58
# e.g. for Pyodide 314.0.x, you need EMSDK version 5.0.3
# the versions can be found in the Makefile.envs file in the Pyodide repo:
# https://github.com/pyodide/pyodide/blob/10b484cfe427e076c929a55dc35cfff01ea8d3bc/Makefile.envs
# https://github.com/pyodide/pyodide/blob/314.0.7/Makefile.envs
./emsdk install <version>
./emsdk activate <version>
source ./emsdk_env.sh
Expand All @@ -49,8 +49,9 @@ versions of emsdk tools.
.. code:: shell

# install Pyodide build tools.
# e.g., for version 0.26 of Pyodide, pyodide-build 0.26 and later work
pip install "pyodide-build>=0.26"
# pyodide-build is versioned independently of Pyodide.
# For Pyodide 314.0.x, pyodide-build 0.39.2 and later work well
pip install "pyodide-build>=0.39.2"

Then build with the ``ninja-release-emscripten`` CMake preset,
like below:
Expand Down
4 changes: 3 additions & 1 deletion python/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,9 @@ if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1")
set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
endif()

if(UNIX)
# On Emscripten, the equivalent $ORIGIN runtime path is added
# through the linker flags in SetupCxxFlags.cmake instead
if(UNIX AND NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
# In the event that we are bundling the shared libraries (e.g. in a
# manylinux1 wheel), we need to set the RPATH of the extensions to the
Expand Down
1 change: 1 addition & 0 deletions python/pyarrow/tests/test_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -2202,6 +2202,7 @@ def test_write_csv_decimal(tmpdir, type_factory):
assert out.column('col').cast(type) == table.column('col')


@pytest.mark.threading
def test_read_csv_gil_deadlock():
# GH-38676
# This test depends on several preconditions:
Expand Down
15 changes: 11 additions & 4 deletions python/scripts/run_emscripten_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def do_GET(self) -> bytes | None:
evt.data.print.forEach((x)=>{window.python_logs.push(x)});
}
}
window.pyworker = new Worker("worker.js");
window.pyworker = new Worker("worker.js", {type: "module"});
window.pyworker.onmessage = capturelogs;
</script>
</head>
Expand All @@ -90,8 +90,8 @@ def do_GET(self) -> bytes | None:
self.copyfile(BytesIO(body), self.wfile)
elif self.path.endswith("/worker.js"):
body = b"""
importScripts("./pyodide.js");
onmessage = async function (e) {
import { loadPyodide } from "./pyodide.mjs";
self.onmessage = async function (e) {
const data = e.data;
if (!self.pyodide) {
self.pyodide = await loadPyodide();
Expand Down Expand Up @@ -148,8 +148,14 @@ class NodeDriver:
import subprocess

def __init__(self, hostname, port):
# handle different syntax for util-linux script and macOS BSD-style script
if sys.platform == "darwin":
script_command = [shutil.which("script"), "-q", "/dev/null",
shutil.which("node")]
else:
script_command = [shutil.which("script"), "-c", shutil.which("node")]
self.process = subprocess.Popen(
[shutil.which("script"), "-c", shutil.which("node")],
script_command,
stdin=subprocess.PIPE,
shell=False,
bufsize=0,
Expand Down Expand Up @@ -320,6 +326,7 @@ def _load_pyarrow_in_runner(driver, wheel_name):
if "pyarrow" not in sys.modules:
await micropip.install("hypothesis")
import pyodide_js as pjs
await pjs.loadPackage("packaging")
await pjs.loadPackage("tzdata")
await pjs.loadPackage("numpy")
await pjs.loadPackage("pandas")
Expand Down
Loading