From 42942550da6f19ed00b51d189730babd6ae67f86 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Wed, 29 Jul 2026 15:41:55 -0700 Subject: [PATCH 1/3] Update [ghstack-poisoned] --- backends/cuda/CMakeLists.txt | 45 ++++++++++++++++ .../runtime/executorch_cuda_backend_lib.cpp | 13 +++++ setup.py | 40 +++++++++++++++ tools/cmake/executorch-wheel-config.cmake | 51 +++++++++++++++++++ 4 files changed, 149 insertions(+) create mode 100644 backends/cuda/runtime/executorch_cuda_backend_lib.cpp diff --git a/backends/cuda/CMakeLists.txt b/backends/cuda/CMakeLists.txt index 0ce48d85e92..515eb9b8fb8 100644 --- a/backends/cuda/CMakeLists.txt +++ b/backends/cuda/CMakeLists.txt @@ -236,3 +236,48 @@ install( EXPORT ExecuTorchTargets DESTINATION lib ) + +# Loadable CUDA backend shared library for the C++ SDK / pip wheel. +# +# aoti_cuda_backend above is a static archive whose static initializer registers +# the "CudaBackend". Today that archive is only force-linked into +# _portable_lib.so for Python. To let a standalone C++ program (or a coalesced +# TensorRT+CUDA .pte) load the CUDA delegate the same way QNN ships +# libqnn_executorch_backend.so, wrap it in a shared library that whole-archives +# aoti_cuda_backend so the registration constructor is retained and runs when +# the .so is loaded. The backend resolves executorch_core / extension_cuda +# symbols from the co-shipped libexecutorch.so and libextension_cuda.so at load +# time, so this stays libtorch-free. +# +# Not built on MSVC: the wheel C++ SDK is Linux only. +if(NOT _cuda_is_msvc_toolchain) + add_library( + executorch_cuda_backend SHARED runtime/executorch_cuda_backend_lib.cpp + ) + # Link the static CUDA backend plainly: aoti_cuda_backend already carries a + # whole-archive INTERFACE link option + # (executorch_target_link_options_shared_lib above), which force-loads it into + # this .so so its "CudaBackend" registration constructor runs on load. Do NOT + # also wrap it in $ here: that would whole-archive + # the same archive twice and run the static allocator/backend registration + # twice, aborting on the duplicate-registration ET_CHECK at load. Link the + # SHARED runtime (executorch_shared -> libexecutorch.so), not the static + # executorch_core, so runtime symbols resolve from the one process-wide + # libexecutorch.so at load (DT_NEEDED) instead of embedding a second copy of + # the registry. + target_link_libraries( + executorch_cuda_backend PUBLIC aoti_cuda_backend executorch_shared + extension_cuda + ) + # Find co-shipped libexecutorch.so / libextension_cuda.so next to this .so in + # the wheel (executorch/lib/), matching QNN's backend rpath approach. + set_target_properties( + executorch_cuda_backend PROPERTIES BUILD_RPATH "$ORIGIN" INSTALL_RPATH + "$ORIGIN" + ) + install( + TARGETS executorch_cuda_backend + EXPORT ExecuTorchTargets + DESTINATION lib + ) +endif() diff --git a/backends/cuda/runtime/executorch_cuda_backend_lib.cpp b/backends/cuda/runtime/executorch_cuda_backend_lib.cpp new file mode 100644 index 00000000000..0def55ee717 --- /dev/null +++ b/backends/cuda/runtime/executorch_cuda_backend_lib.cpp @@ -0,0 +1,13 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +// This translation unit intentionally contains no symbols. The +// executorch_cuda_backend shared library exists solely to bundle the +// whole-archived aoti_cuda_backend static library so its "CudaBackend" +// registration constructor is retained and runs when the .so is loaded. CMake +// requires a SHARED target to have at least one source file. diff --git a/setup.py b/setup.py index 0ce81ae028f..42c03c6eefd 100644 --- a/setup.py +++ b/setup.py @@ -864,6 +864,12 @@ def run(self): "extension/named_data_map/merged_data_map.h", "extension/memory_allocator/malloc_memory_allocator.h", "extension/memory_allocator/memory_allocator_utils.h", + # CUDA caller-stream extension headers. Harmless on a CPU + # wheel (headers only); the matching libextension_cuda.so and + # the executorch::extension_cuda / executorch::cuda_backend + # CMake targets are shipped/defined only in a CUDA wheel. + "extension/cuda/caller_stream.h", + "extension/cuda/export.h", ] if sys.platform == "linux" else [] @@ -1110,6 +1116,11 @@ def run(self): # noqa C901 if cmake_cache.is_enabled("EXECUTORCH_BUILD_CUDA"): cmake_build_args += ["--target", "aoti_cuda_backend"] cmake_build_args += ["--target", "aoti_common_shims_slim"] + # Loadable CUDA delegate + caller-stream shared libs for the C++ + # SDK, built only for a wheel that also builds the shared runtime. + if cmake_cache.is_enabled("EXECUTORCH_BUILD_SHARED"): + cmake_build_args += ["--target", "executorch_cuda_backend"] + cmake_build_args += ["--target", "extension_cuda"] if cmake_cache.is_enabled("EXECUTORCH_BUILD_EXTENSION_MODULE"): cmake_build_args += ["--target", "extension_module"] @@ -1293,6 +1304,35 @@ def run(self): # noqa C901 if sys.platform == "linux" else [] ), + # CUDA delegate binaries for the C++ SDK, shipped only in a + # CUDA-enabled wheel (PyTorch-style: the default wheel has the core + # runtime, the CUDA-index wheel adds these). Co-located with + # libexecutorch.so in executorch/lib/ so a C++ consumer, or a + # coalesced TensorRT+CUDA .pte, can load and register the CUDA + # delegate. executorch_cuda_backend whole-archives the backend so + # its "CudaBackend" registration runs on load; extension_cuda + # carries the single process-wide caller-stream TLS. Both are + # unversioned .so, so a plain dynamic-lib copy is enough. + *( + [ + BuiltFile( + src_dir="%CMAKE_CACHE_DIR%/backends/cuda/", + src_name="executorch_cuda_backend", + dst="executorch/lib/", + is_dynamic_lib=True, + dependent_cmake_flags=["EXECUTORCH_BUILD_CUDA"], + ), + BuiltFile( + src_dir="%CMAKE_CACHE_DIR%/extension/cuda/", + src_name="extension_cuda", + dst="executorch/lib/", + is_dynamic_lib=True, + dependent_cmake_flags=["EXECUTORCH_BUILD_CUDA"], + ), + ] + if sys.platform == "linux" + else [] + ), ] ), ], diff --git a/tools/cmake/executorch-wheel-config.cmake b/tools/cmake/executorch-wheel-config.cmake index de53a5b12bd..5575e3f9248 100644 --- a/tools/cmake/executorch-wheel-config.cmake +++ b/tools/cmake/executorch-wheel-config.cmake @@ -172,4 +172,55 @@ if(_executorch_shared_LIBRARY) ) endif() endforeach() + + # CUDA delegate targets. Present only in a CUDA-enabled wheel, so each target + # is defined only when its shared library is shipped in executorch/lib/. These + # are real separate shared libraries (not aliases of libexecutorch.so): + # extension_cuda holds the one process-wide caller-stream TLS, and + # cuda_backend whole-archives the CUDA delegate so loading it registers + # "CudaBackend" into the runtime. + find_library( + _executorch_extension_cuda_LIBRARY + NAMES extension_cuda + PATHS "${_executorch_sdk_libdir}" + NO_DEFAULT_PATH + ) + if(_executorch_extension_cuda_LIBRARY AND NOT TARGET + executorch::extension_cuda + ) + # caller_stream.h includes and the real target links + # CUDA::cudart PUBLIC, so reproduce that usage requirement. + include(CMakeFindDependencyMacro) + find_dependency(CUDAToolkit) + add_library(executorch::extension_cuda SHARED IMPORTED) + set_target_properties( + executorch::extension_cuda + PROPERTIES IMPORTED_LOCATION "${_executorch_extension_cuda_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${EXECUTORCH_INCLUDE_DIRS}" + INTERFACE_COMPILE_FEATURES cxx_std_17 + INTERFACE_LINK_LIBRARIES CUDA::cudart + ) + endif() + + find_library( + _executorch_cuda_backend_LIBRARY + NAMES executorch_cuda_backend + PATHS "${_executorch_sdk_libdir}" + NO_DEFAULT_PATH + ) + if(_executorch_cuda_backend_LIBRARY AND NOT TARGET executorch::cuda_backend) + add_library(executorch::cuda_backend SHARED IMPORTED) + set_target_properties( + executorch::cuda_backend + PROPERTIES IMPORTED_LOCATION "${_executorch_cuda_backend_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${EXECUTORCH_INCLUDE_DIRS}" + INTERFACE_COMPILE_FEATURES cxx_std_17 + ) + set_property( + TARGET executorch::cuda_backend + APPEND + PROPERTY INTERFACE_LINK_LIBRARIES executorch::runtime + executorch::extension_cuda + ) + endif() endif() From 72a965638a758214504a1d5ad629dfe834411436 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Wed, 29 Jul 2026 18:27:50 -0700 Subject: [PATCH 2/3] Update [ghstack-poisoned] --- backends/cuda/CMakeLists.txt | 8 ++++++++ tools/cmake/executorch-wheel-config.cmake | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/backends/cuda/CMakeLists.txt b/backends/cuda/CMakeLists.txt index a0ee2744dd6..25616837f94 100644 --- a/backends/cuda/CMakeLists.txt +++ b/backends/cuda/CMakeLists.txt @@ -290,6 +290,14 @@ if(NOT _cuda_is_msvc_toolchain AND EXECUTORCH_BUILD_SHARED) PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH "$ORIGIN:$ORIGIN/../backends/cuda" ) + # Force-load for consumers of the EXPORTED target (from-source installs via + # ExecuTorchTargets), matching how QNN ships its backend. This library only + # has load-time registration side effects and no referenced symbols, so under + # an --as-needed-default toolchain a consumer linking the bare target would + # otherwise drop it and CudaBackend would never register. The prebuilt + # wheel-config imported target carries an equivalent --no-as-needed anchor; + # this covers the source-export path. + executorch_target_link_options_shared_lib(executorch_cuda_backend) install( TARGETS executorch_cuda_backend EXPORT ExecuTorchTargets diff --git a/tools/cmake/executorch-wheel-config.cmake b/tools/cmake/executorch-wheel-config.cmake index 72937f31ad8..d9684019f99 100644 --- a/tools/cmake/executorch-wheel-config.cmake +++ b/tools/cmake/executorch-wheel-config.cmake @@ -208,6 +208,19 @@ if(_executorch_shared_LIBRARY) INTERFACE_COMPILE_FEATURES cxx_std_17 INTERFACE_LINK_LIBRARIES CUDA::cudart ) + else() + # The CUDA delegate libraries are shipped in this wheel, but no CUDA + # development toolkit was found, so the executorch::extension_cuda / + # executorch::cuda_backend targets are intentionally not defined. Warn so + # a consumer that expected them gets a clear reason instead of an opaque + # "unknown target executorch::cuda_backend" error later. + message( + WARNING + "ExecuTorch: the CUDA delegate libraries are present in this wheel " + "(${_executorch_extension_cuda_LIBRARY}), but find_package(CUDAToolkit) " + "failed, so the executorch::extension_cuda and executorch::cuda_backend " + "targets are NOT defined. Install the CUDA toolkit to use them." + ) endif() endif() From 0a56bbf2b6d53105eb90b9500dc9595929b1b36a Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Wed, 29 Jul 2026 18:43:23 -0700 Subject: [PATCH 3/3] Update [ghstack-poisoned] --- .github/workflows/pull.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 27c2906c9ff..3b26d8f4c1a 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -139,6 +139,34 @@ jobs: # shipped libexecutorch.so via find_package(executorch). PYTHON_EXECUTABLE=python bash .ci/scripts/test_cpp_sdk_wheel.sh + test-cpp-sdk-wheel-cuda-linux: + name: test-cpp-sdk-wheel-cuda-linux + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + with: + # GPU runner so nvcc is present: the wheel build auto-detects CUDA and + # ships the CUDA delegate, and test_cpp_sdk_wheel.sh then exercises the + # real libexecutorch_cuda_backend.so (dlopen + CudaBackend registration). + # On the CPU job above this same script cleanly skips the CUDA check. + runner: linux.g5.4xlarge.nvidia.gpu + gpu-arch-type: cuda + gpu-arch-version: "12.6" + use-custom-docker-registry: false + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + set -eux + # The generic Linux job chooses to use base env, not the one setup by the image + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + + PYTHON_EXECUTABLE=python bash .ci/scripts/test_cpp_sdk_wheel.sh + test-models-linux-basic: name: test-models-linux-basic uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main