From 31f98c5d0383e3d98b77dec1429f90c48ac29ecd Mon Sep 17 00:00:00 2001 From: kvmto Date: Fri, 26 Jun 2026 15:48:57 +0000 Subject: [PATCH 01/16] Add hardware pinning (cuda_device_id/numa_node_id) for QEC decoders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow any registered decoder to be assigned to specific hardware through both decoder kwargs and the realtime YAML configuration: - cuda_device_id: GPU decoders - numa_node_id: CPU decoders Placement is applied generically at the decoder construction entry point (decoder::get), so every registered decoder—including third-party plugins, regardless of how their creator is implemented—is constructed on the requested hardware. GPU resources are created on cuda_device_id, while CPU decoder allocations first-touch the requested NUMA node. Negative or out-of-range ids are rejected; omitted fields are a no-op. Add public utilities for decoder authors: - device_affinity.h (CUDA-free): read_cuda_device_id(), read_numa_node_id(), and ScopedNumaNode, implemented using raw Linux syscalls (set_mempolicy + sched_setaffinity), Linux-guarded with a no-op fallback and no libnuma dependency. - scoped_cuda_device.h: ScopedCudaDevice RAII helper for temporarily setting and restoring the CUDA device. The public decoder.h interface remains free of CUDA and NUMA dependencies. Support decode-time device re-assertion for decoders whose decode path may execute on a different thread than construction (for example, decode_async()). trt_decoder now uses ScopedCudaDevice for this purpose, and other GPU decoders can opt in using the same public helper. Wire both placement fields through the realtime configuration pipeline, including the configuration structs, YAML serialization, Python bindings, and runtime parameter injection. Add tests covering: - device_affinity readers and ScopedNumaNode restoration - ScopedCudaDevice placement verified with cudaPointerGetAttributes() - generic decoder::get() validation for invalid hardware ids - trt_decoder invalid-device rejection and decode_async placement - YAML round-trip and runtime parameter injection This change intentionally does not address nv-qldpc-decoder runtime device handling or realtime-session/ring NUMA IPC pinning, which require separate follow-up work. Signed-off-by: kvmto --- libs/qec/include/cudaq/qec/device_affinity.h | 138 ++++++++++++++++++ .../cudaq/qec/realtime/decoding_config.h | 2 + .../include/cudaq/qec/scoped_cuda_device.h | 58 ++++++++ libs/qec/lib/decoder.cpp | 4 + .../plugins/trt_decoder/trt_decoder.cpp | 10 ++ libs/qec/lib/realtime/config.cpp | 2 + libs/qec/lib/realtime/realtime_decoding.cpp | 4 + .../python/bindings/py_decoding_config.cpp | 2 + libs/qec/unittests/CMakeLists.txt | 5 + .../decoders/trt_decoder/test_trt_decoder.cpp | 108 ++++++++++++++ libs/qec/unittests/test_decoders.cpp | 20 +++ libs/qec/unittests/test_decoders_yaml.cpp | 38 +++++ libs/qec/unittests/test_device_affinity.cpp | 81 ++++++++++ 13 files changed, 472 insertions(+) create mode 100644 libs/qec/include/cudaq/qec/device_affinity.h create mode 100644 libs/qec/include/cudaq/qec/scoped_cuda_device.h create mode 100644 libs/qec/unittests/test_device_affinity.cpp diff --git a/libs/qec/include/cudaq/qec/device_affinity.h b/libs/qec/include/cudaq/qec/device_affinity.h new file mode 100644 index 000000000..816ced6b2 --- /dev/null +++ b/libs/qec/include/cudaq/qec/device_affinity.h @@ -0,0 +1,138 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ + +#pragma once + +#include "cuda-qx/core/heterogeneous_map.h" +#include +#include + +#if defined(__linux__) +#include +#include +#include +#include +#include +#include +#else +#include +#endif + +namespace cudaq::qec { + +namespace detail { + +inline int read_pin_key(const cudaqx::heterogeneous_map ¶ms, + const std::string &key) { + if (!params.contains(key)) + return -1; + // kwargs stores Python int as std::size_t; YAML stores std::optional. + // heterogeneous_map::get resolves both via the related-types fallback. + int value = params.get(key); + if (value < 0) + throw std::runtime_error(key + " must be >= 0 (got " + + std::to_string(value) + ")"); + return value; +} + +} // namespace detail + +/// @brief GPU device id for a decoder. -1 = inherit current device. +inline int read_cuda_device_id(const cudaqx::heterogeneous_map ¶ms) { + return detail::read_pin_key(params, "cuda_device_id"); +} + +/// @brief NUMA node id for a decoder. -1 = no binding. +inline int read_numa_node_id(const cudaqx::heterogeneous_map ¶ms) { + return detail::read_pin_key(params, "numa_node_id"); +} + +/// @brief Bind the calling thread + its allocations to a NUMA node for the +/// lifetime of the object. node < 0 is a no-op. Linux only. +class ScopedNumaNode { +public: + explicit ScopedNumaNode(int node) { + if (node < 0) + return; +#if defined(__linux__) + CPU_ZERO(&previous_set_); + if (sched_getaffinity(0, sizeof(cpu_set_t), &previous_set_) == 0) + has_previous_affinity_ = true; + + cpu_set_t node_set; + CPU_ZERO(&node_set); + if (build_node_cpuset(node, node_set)) { + sched_setaffinity(0, sizeof(cpu_set_t), &node_set); + affinity_set_ = true; + } + + if (node < static_cast(sizeof(unsigned long) * 8)) { + unsigned long nodemask = 1UL << node; + syscall(SYS_set_mempolicy, MPOL_BIND, &nodemask, + static_cast(sizeof(nodemask) * 8)); + mempolicy_set_ = true; + } +#else + static bool warned = false; + if (!warned) { + std::cerr << "[cudaq-qec] numa_node_id ignored: NUMA binding is only " + "supported on Linux." + << std::endl; + warned = true; + } +#endif + } + + ~ScopedNumaNode() { +#if defined(__linux__) + if (mempolicy_set_) + syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); + if (affinity_set_ && has_previous_affinity_) + sched_setaffinity(0, sizeof(cpu_set_t), &previous_set_); +#endif + } + + ScopedNumaNode(const ScopedNumaNode &) = delete; + ScopedNumaNode &operator=(const ScopedNumaNode &) = delete; + +private: +#if defined(__linux__) + // Parse /sys/devices/system/node/node/cpulist (e.g. "0-7,16-23"). + static bool build_node_cpuset(int node, cpu_set_t &out) { + std::ifstream f("/sys/devices/system/node/node" + std::to_string(node) + + "/cpulist"); + if (!f.is_open()) + return false; + std::string list; + std::getline(f, list); + if (list.empty()) + return false; + std::stringstream ss(list); + std::string range; + bool any = false; + while (std::getline(ss, range, ',')) { + auto dash = range.find('-'); + int lo = std::stoi(range.substr(0, dash)); + int hi = (dash == std::string::npos) ? lo + : std::stoi(range.substr(dash + 1)); + for (int c = lo; c <= hi; ++c) { + CPU_SET(c, &out); + any = true; + } + } + return any; + } + + cpu_set_t previous_set_; + bool has_previous_affinity_ = false; + bool affinity_set_ = false; + bool mempolicy_set_ = false; +#endif +}; + +} // namespace cudaq::qec diff --git a/libs/qec/include/cudaq/qec/realtime/decoding_config.h b/libs/qec/include/cudaq/qec/realtime/decoding_config.h index d8687f8ba..6314af518 100644 --- a/libs/qec/include/cudaq/qec/realtime/decoding_config.h +++ b/libs/qec/include/cudaq/qec/realtime/decoding_config.h @@ -166,6 +166,8 @@ struct decoder_config { std::vector H_sparse; std::vector O_sparse; std::vector D_sparse; + std::optional cuda_device_id; + std::optional numa_node_id; std::variant diff --git a/libs/qec/include/cudaq/qec/scoped_cuda_device.h b/libs/qec/include/cudaq/qec/scoped_cuda_device.h new file mode 100644 index 000000000..f3c9284c2 --- /dev/null +++ b/libs/qec/include/cudaq/qec/scoped_cuda_device.h @@ -0,0 +1,58 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ + +#pragma once + +#include +#include +#include + +namespace cudaq::qec { + +/// @brief RAII guard that sets the calling thread's CUDA current device and +/// restores the previous device on scope exit. Decoder implementations that +/// allocate GPU resources or launch GPU work should instantiate this guard in +/// their constructor and in any decode path that uses the GPU. +/// +/// target < 0 = inherit the current device (no-op). +/// Throws std::runtime_error if target is out of range. +class ScopedCudaDevice { +public: + explicit ScopedCudaDevice(int target) { + if (target < 0) + return; + int count = 0; + if (cudaGetDeviceCount(&count) != cudaSuccess || target >= count) + throw std::runtime_error( + "cuda_device_id " + std::to_string(target) + + " out of range (device_count=" + std::to_string(count) + ")"); + if (cudaGetDevice(&previous_) != cudaSuccess) + throw std::runtime_error("cudaGetDevice failed before device switch"); + // No switch needed when already on the target device; nothing to restore. + if (previous_ != target) { + if (cudaSetDevice(target) != cudaSuccess) + throw std::runtime_error("cudaSetDevice failed for device " + + std::to_string(target)); + active_ = true; + } + } + + ~ScopedCudaDevice() { + if (active_) + cudaSetDevice(previous_); + } + + ScopedCudaDevice(const ScopedCudaDevice &) = delete; + ScopedCudaDevice &operator=(const ScopedCudaDevice &) = delete; + +private: + int previous_ = -1; + bool active_ = false; +}; + +} // namespace cudaq::qec diff --git a/libs/qec/lib/decoder.cpp b/libs/qec/lib/decoder.cpp index 6e95a776c..ef992901e 100644 --- a/libs/qec/lib/decoder.cpp +++ b/libs/qec/lib/decoder.cpp @@ -9,7 +9,9 @@ #include "cudaq/qec/decoder.h" #include "common/FmtCore.h" #include "cuda-qx/core/library_utils.h" +#include "cudaq/qec/device_affinity.h" #include "cudaq/qec/plugin_loader.h" +#include "cudaq/qec/scoped_cuda_device.h" #include "cudaq/qec/version.h" #include "cudaq/runtime/logger/logger.h" #include @@ -138,6 +140,8 @@ decoder::get(const std::string &name, const decoder_init &init, "invalid decoder requested: " + name + ". Run with CUDAQ_LOG_LEVEL=info (environment variable) to see " "additional plugin diagnostics at startup."); + ScopedCudaDevice device_guard(read_cuda_device_id(param_map)); + ScopedNumaNode numa_guard(read_numa_node_id(param_map)); return iter->second(init, param_map); } diff --git a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp index 5e3779661..1ebd260fb 100644 --- a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp +++ b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp @@ -7,7 +7,9 @@ ******************************************************************************/ #include "cudaq/qec/decoder.h" +#include "cudaq/qec/device_affinity.h" #include "cudaq/qec/trt_decoder_internal.h" +#include "cudaq/qec/scoped_cuda_device.h" #include "cudaq/runtime/logger/logger.h" #include #include @@ -401,6 +403,9 @@ class trt_decoder : public decoder { // True when decoder is fully configured and ready for inference bool decoder_ready_ = false; + // Target CUDA device for this decoder's resources/decodes. -1 = current. + int cuda_device_id_ = -1; + // Batch dimension from TensorRT model (first dimension of input tensor) size_t model_batch_size_ = 1; @@ -544,6 +549,8 @@ trt_decoder::trt_decoder(const cudaq::qec::sparse_binary_matrix &H, impl_ = std::make_unique(); + cuda_device_id_ = cudaq::qec::read_cuda_device_id(params); + try { // Validate parameters trt_decoder_internal::validate_trt_decoder_parameters(params); @@ -928,6 +935,9 @@ size_t trt_decoder::failure_result_size() const { template std::vector trt_decoder::decode_batch_impl( const std::vector> &syndromes) const { + // All decode entry points (decode, decode_batch, decode_async) funnel here, + // so the device is re-asserted for inference regardless of the calling thread. + cudaq::qec::ScopedCudaDevice device_guard(cuda_device_id_); std::vector results; results.reserve(syndromes.size()); diff --git a/libs/qec/lib/realtime/config.cpp b/libs/qec/lib/realtime/config.cpp index c8737d7cf..5392e597b 100644 --- a/libs/qec/lib/realtime/config.cpp +++ b/libs/qec/lib/realtime/config.cpp @@ -534,6 +534,8 @@ struct MappingTraits { io.mapRequired("H_sparse", config.H_sparse); io.mapRequired("O_sparse", config.O_sparse); io.mapRequired("D_sparse", config.D_sparse); + io.mapOptional("cuda_device_id", config.cuda_device_id); + io.mapOptional("numa_node_id", config.numa_node_id); // Validate that the number of rows in the H_sparse vector is equal to // syndrome_size. diff --git a/libs/qec/lib/realtime/realtime_decoding.cpp b/libs/qec/lib/realtime/realtime_decoding.cpp index f1402c307..c28f3e53b 100644 --- a/libs/qec/lib/realtime/realtime_decoding.cpp +++ b/libs/qec/lib/realtime/realtime_decoding.cpp @@ -183,6 +183,10 @@ namespace cudaq::qec::decoding::host { cudaqx::heterogeneous_map prepare_decoder_params( const cudaq::qec::decoding::config::decoder_config &decoder_config) { auto params = decoder_config.decoder_custom_args_to_heterogeneous_map(); + if (decoder_config.cuda_device_id.has_value()) + params.insert("cuda_device_id", decoder_config.cuda_device_id.value()); + if (decoder_config.numa_node_id.has_value()) + params.insert("numa_node_id", decoder_config.numa_node_id.value()); if (decoder_config.type != "trt_decoder") return params; diff --git a/libs/qec/python/bindings/py_decoding_config.cpp b/libs/qec/python/bindings/py_decoding_config.cpp index 3bc3c454e..80d9a7ae4 100644 --- a/libs/qec/python/bindings/py_decoding_config.cpp +++ b/libs/qec/python/bindings/py_decoding_config.cpp @@ -241,6 +241,8 @@ void bindDecodingConfig(nb::module_ &mod) { .def_rw("H_sparse", &decoder_config::H_sparse) .def_rw("O_sparse", &decoder_config::O_sparse) .def_rw("D_sparse", &decoder_config::D_sparse) + .def_rw("cuda_device_id", &decoder_config::cuda_device_id) + .def_rw("numa_node_id", &decoder_config::numa_node_id) .def_rw("decoder_custom_args", &decoder_config::decoder_custom_args) .def( "set_decoder_custom_args", diff --git a/libs/qec/unittests/CMakeLists.txt b/libs/qec/unittests/CMakeLists.txt index 406431f7e..a1ee8ed0f 100644 --- a/libs/qec/unittests/CMakeLists.txt +++ b/libs/qec/unittests/CMakeLists.txt @@ -49,6 +49,11 @@ target_link_libraries(test_decoders_yaml PRIVATE add_dependencies(CUDAQXQECUnitTests test_decoders_yaml) gtest_discover_tests(test_decoders_yaml) +add_executable(test_device_affinity test_device_affinity.cpp) +target_link_libraries(test_device_affinity PRIVATE GTest::gtest_main cudaq-qec cudaq::cudaq) +add_dependencies(CUDAQXQECUnitTests test_device_affinity) +gtest_discover_tests(test_device_affinity) + add_executable(test_qec test_qec.cpp) target_link_libraries(test_qec PRIVATE GTest::gtest_main cudaq-qec cudaq::cudaq-stim-target) add_dependencies(CUDAQXQECUnitTests test_qec) diff --git a/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp b/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp index 205bf6f2c..d679a98d8 100644 --- a/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp +++ b/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp @@ -9,6 +9,7 @@ #include "trt_test_data.h" #include "cudaq/qec/decoder.h" #include "cudaq/qec/trt_decoder_internal.h" +#include "cudaq/qec/scoped_cuda_device.h" #include #include #include @@ -739,3 +740,110 @@ TEST_F(TRTDecoderTest, CompositeGlobalDecoderCombinesLogicalFrame) { // require actual TensorRT/CUDA initialization which is not available in the // test environment. Only parameter validation and utility function tests are // enabled above. + +TEST_F(TRTDecoderTest, CudaDeviceId_OutOfRangeThrows) { + if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; + std::string onnx_path = get_onnx_asset_path(); + if (!std::filesystem::exists(onnx_path)) + GTEST_SKIP() << "ONNX model not found: " << onnx_path; + int count = 0; + cudaGetDeviceCount(&count); + cudaqx::tensor H_small({1, 1}); + cudaqx::heterogeneous_map params; + params.insert("onnx_load_path", onnx_path); + params.insert("cuda_device_id", count); // one past the last valid device + EXPECT_THROW( + { cudaq::qec::decoder::get("trt_decoder", H_small, params); }, + std::runtime_error); +} + +TEST_F(TRTDecoderTest, CudaDeviceId_DecodeAsyncOnGpu1) { + if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; + int count = 0; + cudaGetDeviceCount(&count); + if (count < 2) GTEST_SKIP() << "needs >= 2 GPUs to prove non-default pinning"; + std::string onnx_path = get_onnx_asset_path(); + if (!std::filesystem::exists(onnx_path)) + GTEST_SKIP() << "ONNX model not found: " << onnx_path; + + cudaSetDevice(0); // calling thread stays on the default device + + std::size_t num_detectors = NUM_DETECTORS; + cudaqx::tensor H_mat({num_detectors, num_detectors}); + for (std::size_t i = 0; i < num_detectors; ++i) + H_mat.at({i, i}) = 1; + + cudaqx::heterogeneous_map params; + params.insert("onnx_load_path", onnx_path); + params.insert("cuda_device_id", 1); // engine/buffers must land on GPU 1 + + std::unique_ptr d; + ASSERT_NO_THROW({ d = cudaq::qec::decoder::get("trt_decoder", H_mat, params); }); + + // decode_async spawns a fresh thread (defaults to GPU 0). If trt did NOT + // re-assert the device, the decode would run on GPU 0 while the engine lives + // on GPU 1 -> error/garbage. A correct result proves the per-decode guard. + std::vector syndrome(TEST_INPUTS[0].begin(), + TEST_INPUTS[0].end()); + auto fut = d->decode_async(syndrome); + cudaq::qec::decoder_result res; + ASSERT_NO_THROW({ res = fut.get(); }); + ASSERT_FALSE(res.result.empty()); + float trt_output = res.result[0]; + float expected_output = TEST_OUTPUTS[0][0]; + float error = std::abs(trt_output - expected_output); + EXPECT_LT(error, 1e-4f) + << "GPU-1 decode differs from expected: got " << trt_output + << ", expected " << expected_output; +} + +TEST(ScopedCudaDevice, NegativeIsNoop) { + if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; + int before = -1; + cudaGetDevice(&before); + { cudaq::qec::ScopedCudaDevice guard(-1); } + int after = -1; + cudaGetDevice(&after); + EXPECT_EQ(before, after); +} + +TEST(ScopedCudaDevice, SetsAndRestores) { + if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; + int count = 0; + cudaGetDeviceCount(&count); + if (count < 2) GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); + { + cudaq::qec::ScopedCudaDevice guard(1); + int cur = -1; + cudaGetDevice(&cur); + EXPECT_EQ(cur, 1); + } + int restored = -1; + cudaGetDevice(&restored); + EXPECT_EQ(restored, 0); +} + +TEST(ScopedCudaDevice, AllocLandsOnTarget) { + if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; + int count = 0; + cudaGetDeviceCount(&count); + if (count < 2) GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); + void *p = nullptr; + { + cudaq::qec::ScopedCudaDevice guard(1); + ASSERT_EQ(cudaMalloc(&p, 1024), cudaSuccess); + } + cudaPointerAttributes attr{}; + ASSERT_EQ(cudaPointerGetAttributes(&attr, p), cudaSuccess); + EXPECT_EQ(attr.device, 1); + cudaFree(p); +} + +TEST(ScopedCudaDevice, OutOfRangeThrows) { + if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; + int count = 0; + cudaGetDeviceCount(&count); + EXPECT_THROW({ cudaq::qec::ScopedCudaDevice guard(count); }, std::runtime_error); +} diff --git a/libs/qec/unittests/test_decoders.cpp b/libs/qec/unittests/test_decoders.cpp index 3cb6cf4b1..d063bf670 100644 --- a/libs/qec/unittests/test_decoders.cpp +++ b/libs/qec/unittests/test_decoders.cpp @@ -226,6 +226,16 @@ TEST(SampleDecoder, RealtimeApiAndDefaultGraphHooks) { EXPECT_FALSE(decoder->enqueue_syndrome(msyn.data(), msyn.size() + 1)); } +TEST(SampleDecoder, AcceptsNumaNodeId) { + std::size_t block_size = 10, syndrome_size = 5; + cudaqx::tensor H({syndrome_size, block_size}); + auto H_sparse = cudaq::qec::sparse_binary_matrix(H); + cudaqx::heterogeneous_map params; + params.insert("numa_node_id", 0); // node 0 always exists + auto d = cudaq::qec::decoder::get("sample_decoder", H_sparse, params); + ASSERT_NE(d, nullptr); +} + TEST(DecoderPlugins, SingleErrorLutExample_DecodesSingletonColumnSyndromes) { using cudaq::qec::float_t; @@ -954,6 +964,16 @@ TEST(StimDemGetDecoder, ThrowsOnProbabilityOutOfRange) { std::runtime_error); } +TEST(CudaDeviceId, OutOfRangeThrowsForAnyDecoder) { + // A device id beyond the available devices is rejected during construction + // for any decoder, including CPU ones. + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + params.insert("cuda_device_id", 999999); + EXPECT_THROW(cudaq::qec::decoder::get("multi_error_lut", H, params), + std::runtime_error); +} + TEST(StimDemGetDecoder, ThrowsOnMalformedStimDem) { EXPECT_THROW(cudaq::qec::get_decoder("single_error_lut", "not a valid DEM"), std::runtime_error); diff --git a/libs/qec/unittests/test_decoders_yaml.cpp b/libs/qec/unittests/test_decoders_yaml.cpp index 1662f2925..344b0979d 100644 --- a/libs/qec/unittests/test_decoders_yaml.cpp +++ b/libs/qec/unittests/test_decoders_yaml.cpp @@ -609,3 +609,41 @@ TEST(DecoderConfigTest, SimulationHostPointerWrappersForwardToHostRuntime) { EXPECT_EQ(corrections, (std::vector{0})); finalize_decoders(); } + +TEST(DecoderYaml, PinFieldsRoundTrip) { + using namespace cudaq::qec::decoding::config; + multi_decoder_config multi_config; + decoder_config cfg; + cfg.id = 0; + cfg.type = "multi_error_lut"; + cfg.block_size = 4; + cfg.syndrome_size = 2; + cfg.H_sparse = {0, -1, 1, -1}; + cfg.O_sparse = {}; + cfg.D_sparse = {}; + cfg.cuda_device_id = 1; + cfg.numa_node_id = 0; + multi_config.decoders.push_back(cfg); + + test_decoder_yaml_roundtrip(multi_config); + + auto reparsed = + multi_decoder_config::from_yaml_str(multi_config.to_yaml_str(200)); + ASSERT_EQ(reparsed.decoders.size(), 1u); + EXPECT_EQ(reparsed.decoders[0].cuda_device_id, 1); + EXPECT_EQ(reparsed.decoders[0].numa_node_id, 0); +} + +TEST(DecoderYaml, PrepareParamsInjectsKeys) { + using namespace cudaq::qec::decoding::config; + decoder_config cfg; + cfg.id = 0; + cfg.type = "multi_error_lut"; + cfg.block_size = 4; + cfg.syndrome_size = 2; + cfg.cuda_device_id = 1; + cfg.numa_node_id = 0; + auto params = cudaq::qec::decoding::host::prepare_decoder_params(cfg); + EXPECT_TRUE(params.contains("cuda_device_id")); + EXPECT_TRUE(params.contains("numa_node_id")); +} diff --git a/libs/qec/unittests/test_device_affinity.cpp b/libs/qec/unittests/test_device_affinity.cpp new file mode 100644 index 000000000..267b8457e --- /dev/null +++ b/libs/qec/unittests/test_device_affinity.cpp @@ -0,0 +1,81 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ + +#include "cudaq/qec/device_affinity.h" +#include + +#if defined(__linux__) +#include +#include +#include +#include +#endif + +using cudaq::qec::read_cuda_device_id; +using cudaq::qec::read_numa_node_id; +using cudaqx::heterogeneous_map; + +TEST(DeviceAffinity, ReadAbsentReturnsMinusOne) { + heterogeneous_map m; + EXPECT_EQ(read_cuda_device_id(m), -1); + EXPECT_EQ(read_numa_node_id(m), -1); +} + +TEST(DeviceAffinity, ReadIntStorage) { // YAML path: std::optional -> int + heterogeneous_map m; + m.insert("cuda_device_id", 3); + m.insert("numa_node_id", 1); + EXPECT_EQ(read_cuda_device_id(m), 3); + EXPECT_EQ(read_numa_node_id(m), 1); +} + +TEST(DeviceAffinity, ReadSizeTStorage) { // kwargs path: Python int -> std::size_t + heterogeneous_map m; + m.insert("cuda_device_id", std::size_t{2}); + EXPECT_EQ(read_cuda_device_id(m), 2); +} + +TEST(DeviceAffinity, ReadNegativeThrows) { + heterogeneous_map m; + m.insert("cuda_device_id", -1); + EXPECT_THROW(read_cuda_device_id(m), std::runtime_error); +} + +TEST(DeviceAffinity, ScopedNumaNodeNegativeIsNoop) { + EXPECT_NO_THROW({ cudaq::qec::ScopedNumaNode guard(-1); }); +} + +TEST(DeviceAffinity, ScopedNumaNodeZeroBinds) { + EXPECT_NO_THROW({ cudaq::qec::ScopedNumaNode guard(0); }); +} + +#if defined(__linux__) +TEST(DeviceAffinity, ScopedNumaNodeSetsAndRestoresMempolicy) { + int mode = -12345; + { + cudaq::qec::ScopedNumaNode guard(0); + long rc = syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL); + ASSERT_EQ(rc, 0); + EXPECT_EQ(mode, MPOL_BIND); + } + long rc = syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL); + ASSERT_EQ(rc, 0); + EXPECT_EQ(mode, MPOL_DEFAULT); +} + +TEST(DeviceAffinity, ScopedNumaNodeRestoresAffinity) { + cpu_set_t before; + CPU_ZERO(&before); + ASSERT_EQ(sched_getaffinity(0, sizeof(before), &before), 0); + { cudaq::qec::ScopedNumaNode guard(0); } + cpu_set_t after; + CPU_ZERO(&after); + ASSERT_EQ(sched_getaffinity(0, sizeof(after), &after), 0); + EXPECT_TRUE(CPU_EQUAL(&before, &after)); +} +#endif From 2ca48a61edee7d0c791b0fb44431e59ba4e52942 Mon Sep 17 00:00:00 2001 From: kvmto Date: Mon, 29 Jun 2026 16:01:53 +0000 Subject: [PATCH 02/16] Redesign hardware pinning: base class owns CUDA/NUMA affinity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move cuda_device_id and numa_node_id into the base decoder class. Private RAII guards (CudaDeviceGuard, NumaGuard) in decoder.cpp apply affinity at construction, decode_batch, and decode_async — no plugin code needed. Raw Linux syscalls, no libnuma. Remove the v1 per-plugin ScopedCudaDevice from trt_decoder; add an inline guard to its decode_batch override which bypasses the base class path. Signed-off-by: kvmto --- libs/qec/include/cudaq/qec/decoder.h | 10 ++ libs/qec/include/cudaq/qec/device_affinity.h | 102 +---------- .../include/cudaq/qec/scoped_cuda_device.h | 58 ------- libs/qec/lib/decoder.cpp | 161 +++++++++++++++++- .../plugins/trt_decoder/trt_decoder.cpp | 30 ++-- libs/qec/unittests/CMakeLists.txt | 3 +- .../decoders/trt_decoder/test_trt_decoder.cpp | 69 ++------ libs/qec/unittests/test_decoders.cpp | 28 +++ libs/qec/unittests/test_device_affinity.cpp | 44 +---- 9 files changed, 230 insertions(+), 275 deletions(-) delete mode 100644 libs/qec/include/cudaq/qec/scoped_cuda_device.h diff --git a/libs/qec/include/cudaq/qec/decoder.h b/libs/qec/include/cudaq/qec/decoder.h index 0a2f4c0c0..b6ee6dff3 100644 --- a/libs/qec/include/cudaq/qec/decoder.h +++ b/libs/qec/include/cudaq/qec/decoder.h @@ -245,6 +245,11 @@ class decoder std::size_t get_block_size() { return block_size; } std::size_t get_syndrome_size() { return syndrome_size; } + /// @brief Store hardware affinity parameters read from the constructor params + /// map. Called by decoder::get() after the plugin constructor returns. + /// Not intended for direct use by plugin authors. + void set_hardware_params(const cudaqx::heterogeneous_map ¶ms); + // -- Begin realtime decoding API -- // Note: all of the current realtime decoding API is designed to be used with @@ -357,6 +362,11 @@ class decoder /// @brief The decoder's D matrix in sparse format std::vector> D_sparse; + /// Target CUDA device for this decoder. -1 = inherit caller's device (no-op). + int cuda_device_id_ = -1; + /// Target NUMA node for this decoder. -1 = no binding. + int numa_node_id_ = -1; + private: decode_result_type result_type_ = decode_result_type::decode_to_errs; }; diff --git a/libs/qec/include/cudaq/qec/device_affinity.h b/libs/qec/include/cudaq/qec/device_affinity.h index 816ced6b2..b911ded89 100644 --- a/libs/qec/include/cudaq/qec/device_affinity.h +++ b/libs/qec/include/cudaq/qec/device_affinity.h @@ -12,37 +12,24 @@ #include #include -#if defined(__linux__) -#include -#include -#include -#include -#include -#include -#else -#include -#endif - namespace cudaq::qec { namespace detail { - +/// Returns -1 if key is absent; throws if stored value is negative. +/// Handles both int (YAML path) and std::size_t (Python kwargs path) storage. inline int read_pin_key(const cudaqx::heterogeneous_map ¶ms, const std::string &key) { if (!params.contains(key)) return -1; - // kwargs stores Python int as std::size_t; YAML stores std::optional. - // heterogeneous_map::get resolves both via the related-types fallback. int value = params.get(key); if (value < 0) throw std::runtime_error(key + " must be >= 0 (got " + std::to_string(value) + ")"); return value; } - } // namespace detail -/// @brief GPU device id for a decoder. -1 = inherit current device. +/// @brief GPU device id for a decoder. -1 = inherit current device (no-op). inline int read_cuda_device_id(const cudaqx::heterogeneous_map ¶ms) { return detail::read_pin_key(params, "cuda_device_id"); } @@ -52,87 +39,4 @@ inline int read_numa_node_id(const cudaqx::heterogeneous_map ¶ms) { return detail::read_pin_key(params, "numa_node_id"); } -/// @brief Bind the calling thread + its allocations to a NUMA node for the -/// lifetime of the object. node < 0 is a no-op. Linux only. -class ScopedNumaNode { -public: - explicit ScopedNumaNode(int node) { - if (node < 0) - return; -#if defined(__linux__) - CPU_ZERO(&previous_set_); - if (sched_getaffinity(0, sizeof(cpu_set_t), &previous_set_) == 0) - has_previous_affinity_ = true; - - cpu_set_t node_set; - CPU_ZERO(&node_set); - if (build_node_cpuset(node, node_set)) { - sched_setaffinity(0, sizeof(cpu_set_t), &node_set); - affinity_set_ = true; - } - - if (node < static_cast(sizeof(unsigned long) * 8)) { - unsigned long nodemask = 1UL << node; - syscall(SYS_set_mempolicy, MPOL_BIND, &nodemask, - static_cast(sizeof(nodemask) * 8)); - mempolicy_set_ = true; - } -#else - static bool warned = false; - if (!warned) { - std::cerr << "[cudaq-qec] numa_node_id ignored: NUMA binding is only " - "supported on Linux." - << std::endl; - warned = true; - } -#endif - } - - ~ScopedNumaNode() { -#if defined(__linux__) - if (mempolicy_set_) - syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); - if (affinity_set_ && has_previous_affinity_) - sched_setaffinity(0, sizeof(cpu_set_t), &previous_set_); -#endif - } - - ScopedNumaNode(const ScopedNumaNode &) = delete; - ScopedNumaNode &operator=(const ScopedNumaNode &) = delete; - -private: -#if defined(__linux__) - // Parse /sys/devices/system/node/node/cpulist (e.g. "0-7,16-23"). - static bool build_node_cpuset(int node, cpu_set_t &out) { - std::ifstream f("/sys/devices/system/node/node" + std::to_string(node) + - "/cpulist"); - if (!f.is_open()) - return false; - std::string list; - std::getline(f, list); - if (list.empty()) - return false; - std::stringstream ss(list); - std::string range; - bool any = false; - while (std::getline(ss, range, ',')) { - auto dash = range.find('-'); - int lo = std::stoi(range.substr(0, dash)); - int hi = (dash == std::string::npos) ? lo - : std::stoi(range.substr(dash + 1)); - for (int c = lo; c <= hi; ++c) { - CPU_SET(c, &out); - any = true; - } - } - return any; - } - - cpu_set_t previous_set_; - bool has_previous_affinity_ = false; - bool affinity_set_ = false; - bool mempolicy_set_ = false; -#endif -}; - } // namespace cudaq::qec diff --git a/libs/qec/include/cudaq/qec/scoped_cuda_device.h b/libs/qec/include/cudaq/qec/scoped_cuda_device.h deleted file mode 100644 index f3c9284c2..000000000 --- a/libs/qec/include/cudaq/qec/scoped_cuda_device.h +++ /dev/null @@ -1,58 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include -#include -#include - -namespace cudaq::qec { - -/// @brief RAII guard that sets the calling thread's CUDA current device and -/// restores the previous device on scope exit. Decoder implementations that -/// allocate GPU resources or launch GPU work should instantiate this guard in -/// their constructor and in any decode path that uses the GPU. -/// -/// target < 0 = inherit the current device (no-op). -/// Throws std::runtime_error if target is out of range. -class ScopedCudaDevice { -public: - explicit ScopedCudaDevice(int target) { - if (target < 0) - return; - int count = 0; - if (cudaGetDeviceCount(&count) != cudaSuccess || target >= count) - throw std::runtime_error( - "cuda_device_id " + std::to_string(target) + - " out of range (device_count=" + std::to_string(count) + ")"); - if (cudaGetDevice(&previous_) != cudaSuccess) - throw std::runtime_error("cudaGetDevice failed before device switch"); - // No switch needed when already on the target device; nothing to restore. - if (previous_ != target) { - if (cudaSetDevice(target) != cudaSuccess) - throw std::runtime_error("cudaSetDevice failed for device " + - std::to_string(target)); - active_ = true; - } - } - - ~ScopedCudaDevice() { - if (active_) - cudaSetDevice(previous_); - } - - ScopedCudaDevice(const ScopedCudaDevice &) = delete; - ScopedCudaDevice &operator=(const ScopedCudaDevice &) = delete; - -private: - int previous_ = -1; - bool active_ = false; -}; - -} // namespace cudaq::qec diff --git a/libs/qec/lib/decoder.cpp b/libs/qec/lib/decoder.cpp index ef992901e..6ffe9e856 100644 --- a/libs/qec/lib/decoder.cpp +++ b/libs/qec/lib/decoder.cpp @@ -11,13 +11,141 @@ #include "cuda-qx/core/library_utils.h" #include "cudaq/qec/device_affinity.h" #include "cudaq/qec/plugin_loader.h" -#include "cudaq/qec/scoped_cuda_device.h" #include "cudaq/qec/version.h" #include "cudaq/runtime/logger/logger.h" #include +#include #include #include #include +#if defined(__linux__) +#include +#include +#include +#include +#include +#include +#endif + +namespace { + +// RAII: sets the calling thread's CUDA current device, restores on destruction. +// target < 0 = no-op. Throws std::runtime_error if target is out of range. +struct CudaDeviceGuard { + int prev_ = -1; + bool active_ = false; + + explicit CudaDeviceGuard(int target) { + if (target < 0) + return; + int count = 0; + if (cudaGetDeviceCount(&count) != cudaSuccess || target >= count) + throw std::runtime_error( + "cuda_device_id " + std::to_string(target) + + " out of range (device_count=" + std::to_string(count) + ")"); + cudaGetDevice(&prev_); + if (prev_ != target) { + cudaSetDevice(target); + active_ = true; + } + } + ~CudaDeviceGuard() { + if (active_) + cudaSetDevice(prev_); + } + CudaDeviceGuard(const CudaDeviceGuard &) = delete; + CudaDeviceGuard &operator=(const CudaDeviceGuard &) = delete; +}; + +#if defined(__linux__) +// Parse /sys/devices/system/node/node/cpulist e.g. "0-7,16-23". +// Returns false if the file is missing or empty. +static bool build_node_cpuset(int node, cpu_set_t &out) { + std::ifstream f("/sys/devices/system/node/node" + std::to_string(node) + + "/cpulist"); + if (!f.is_open()) + return false; + std::string list; + std::getline(f, list); + if (list.empty()) + return false; + std::stringstream ss(list); + std::string range; + bool any = false; + while (std::getline(ss, range, ',')) { + auto dash = range.find('-'); + int lo = std::stoi(range.substr(0, dash)); + int hi = + (dash == std::string::npos) ? lo : std::stoi(range.substr(dash + 1)); + for (int c = lo; c <= hi; ++c) { + if (c < CPU_SETSIZE) { // guard against OOB on >1024-CPU machines + CPU_SET(c, &out); + any = true; + } + } + } + return any; +} + +// RAII: binds the calling thread's CPU affinity and memory policy to a NUMA +// node, restores on destruction. node < 0 = no-op. +// Uses raw Linux syscalls -- no libnuma dependency. +struct NumaGuard { + bool mempol_set_ = false; + bool affinity_set_ = false; + bool has_prev_affinity_ = false; + cpu_set_t prev_set_{}; + + explicit NumaGuard(int node) { + if (node < 0) + return; + CPU_ZERO(&prev_set_); + if (sched_getaffinity(0, sizeof(prev_set_), &prev_set_) == 0) + has_prev_affinity_ = true; + + cpu_set_t node_set; + CPU_ZERO(&node_set); + // Only pin CPU affinity when we can restore it; avoids permanent pinning + // if sched_getaffinity failed (e.g., container with locked cpuset). + if (has_prev_affinity_ && build_node_cpuset(node, node_set)) { + if (sched_setaffinity(0, sizeof(node_set), &node_set) == 0) + affinity_set_ = true; + } + + if (node < static_cast(sizeof(unsigned long) * 8)) { + unsigned long nodemask = 1UL << node; + if (syscall(SYS_set_mempolicy, MPOL_BIND, &nodemask, + static_cast(sizeof(nodemask) * 8)) == 0) + mempol_set_ = true; + } + } + + ~NumaGuard() { + if (mempol_set_) + syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); + if (affinity_set_) + sched_setaffinity(0, sizeof(prev_set_), &prev_set_); + } + + NumaGuard(const NumaGuard &) = delete; + NumaGuard &operator=(const NumaGuard &) = delete; +}; +#else +struct NumaGuard { + explicit NumaGuard(int node) { + if (node < 0) + return; + static bool warned = false; + if (!warned) { + std::cerr << "[cudaq-qec] numa_node_id ignored: NUMA binding is only " + "supported on Linux.\n"; + warned = true; + } + } +}; +#endif + +} // anonymous namespace INSTANTIATE_REGISTRY(cudaq::qec::decoder, const cudaq::qec::decoder_init &, const cudaqx::heterogeneous_map &) @@ -89,6 +217,11 @@ decoder::decoder(cudaq::qec::sparse_binary_matrix H) pimpl->should_log = ch[0] == '1' || ch[0] == 'y' || ch[0] == 'Y'; } +void decoder::set_hardware_params(const cudaqx::heterogeneous_map ¶ms) { + cuda_device_id_ = cudaq::qec::read_cuda_device_id(params); + numa_node_id_ = cudaq::qec::read_numa_node_id(params); +} + // Provide a trivial implementation of for tensor decode call. Child // classes should override this if they never want to pass through floats. decoder_result decoder::decode(const cudaqx::tensor &syndrome) { @@ -109,6 +242,10 @@ decoder_result decoder::decode(const cudaqx::tensor &syndrome) { // should override this if they can do it more efficiently than this. std::vector decoder::decode_batch(const std::vector> &syndrome) { + // Apply affinity once for the whole batch (not per-syndrome) to avoid + // repeated syscall overhead on the hot path. + CudaDeviceGuard dev(cuda_device_id_); + NumaGuard numa(numa_node_id_); std::vector result; result.reserve(syndrome.size()); for (auto &s : syndrome) @@ -125,8 +262,15 @@ std::string decoder::get_version() const { std::future decoder::decode_async(const std::vector &syndrome) { - return std::async(std::launch::async, - [this, syndrome] { return this->decode(syndrome); }); + // Capture by value: avoids a data race if the decoder is destroyed before + // the future resolves. + const int cuda_id = cuda_device_id_; + const int numa_id = numa_node_id_; + return std::async(std::launch::async, [this, syndrome, cuda_id, numa_id] { + CudaDeviceGuard dev(cuda_id); + NumaGuard numa(numa_id); + return this->decode(syndrome); + }); } std::unique_ptr @@ -140,9 +284,14 @@ decoder::get(const std::string &name, const decoder_init &init, "invalid decoder requested: " + name + ". Run with CUDAQ_LOG_LEVEL=info (environment variable) to see " "additional plugin diagnostics at startup."); - ScopedCudaDevice device_guard(read_cuda_device_id(param_map)); - ScopedNumaNode numa_guard(read_numa_node_id(param_map)); - return iter->second(init, param_map); + // Guards during construction so allocations land on the right hardware. + // Restored before this function returns; decode-time affinity is re-applied + // per call in decode_batch() and decode_async(). + CudaDeviceGuard ctor_dev(cudaq::qec::read_cuda_device_id(param_map)); + NumaGuard ctor_numa(cudaq::qec::read_numa_node_id(param_map)); + auto d = iter->second(init, param_map); + d->set_hardware_params(param_map); + return d; } namespace details { diff --git a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp index 1ebd260fb..a46b00943 100644 --- a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp +++ b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp @@ -7,9 +7,7 @@ ******************************************************************************/ #include "cudaq/qec/decoder.h" -#include "cudaq/qec/device_affinity.h" #include "cudaq/qec/trt_decoder_internal.h" -#include "cudaq/qec/scoped_cuda_device.h" #include "cudaq/runtime/logger/logger.h" #include #include @@ -403,9 +401,6 @@ class trt_decoder : public decoder { // True when decoder is fully configured and ready for inference bool decoder_ready_ = false; - // Target CUDA device for this decoder's resources/decodes. -1 = current. - int cuda_device_id_ = -1; - // Batch dimension from TensorRT model (first dimension of input tensor) size_t model_batch_size_ = 1; @@ -549,8 +544,6 @@ trt_decoder::trt_decoder(const cudaq::qec::sparse_binary_matrix &H, impl_ = std::make_unique(); - cuda_device_id_ = cudaq::qec::read_cuda_device_id(params); - try { // Validate parameters trt_decoder_internal::validate_trt_decoder_parameters(params); @@ -886,6 +879,26 @@ decoder_result trt_decoder::decode(const std::vector &syndrome) { std::vector trt_decoder::decode_batch(const std::vector> &syndromes) { + // This override bypasses decoder::decode_batch()'s CudaDeviceGuard; apply it + // here so TRT inference lands on the right device regardless of the caller. + int _prev_dev = -1; + bool _dev_switched = false; + if (cuda_device_id_ >= 0) { + cudaGetDevice(&_prev_dev); + if (_prev_dev != cuda_device_id_) { + cudaSetDevice(cuda_device_id_); + _dev_switched = true; + } + } + struct _RestoreDevice { + int prev; + bool active; + ~_RestoreDevice() { + if (active) + cudaSetDevice(prev); + } + } _dev_guard{_prev_dev, _dev_switched}; + // Validate that we have syndromes to decode if (syndromes.empty()) { return {}; @@ -935,9 +948,6 @@ size_t trt_decoder::failure_result_size() const { template std::vector trt_decoder::decode_batch_impl( const std::vector> &syndromes) const { - // All decode entry points (decode, decode_batch, decode_async) funnel here, - // so the device is re-asserted for inference regardless of the calling thread. - cudaq::qec::ScopedCudaDevice device_guard(cuda_device_id_); std::vector results; results.reserve(syndromes.size()); diff --git a/libs/qec/unittests/CMakeLists.txt b/libs/qec/unittests/CMakeLists.txt index a1ee8ed0f..1435c1bf4 100644 --- a/libs/qec/unittests/CMakeLists.txt +++ b/libs/qec/unittests/CMakeLists.txt @@ -35,7 +35,8 @@ find_package(CUDAToolkit REQUIRED) add_compile_options(-Wno-attributes) add_executable(test_decoders test_decoders.cpp decoders/sample_decoder.cpp) -target_link_libraries(test_decoders PRIVATE GTest::gtest_main cudaq-qec cudaq-qec-realtime-decoding cudaq::cudaq libstim) +target_link_libraries(test_decoders PRIVATE GTest::gtest_main cudaq-qec cudaq-qec-realtime-decoding cudaq::cudaq libstim CUDA::cudart) +target_include_directories(test_decoders PRIVATE ${CUDAToolkit_INCLUDE_DIRS}) add_dependencies(CUDAQXQECUnitTests test_decoders) gtest_discover_tests(test_decoders) diff --git a/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp b/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp index d679a98d8..1a9f24113 100644 --- a/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp +++ b/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp @@ -9,7 +9,6 @@ #include "trt_test_data.h" #include "cudaq/qec/decoder.h" #include "cudaq/qec/trt_decoder_internal.h" -#include "cudaq/qec/scoped_cuda_device.h" #include #include #include @@ -742,7 +741,8 @@ TEST_F(TRTDecoderTest, CompositeGlobalDecoderCombinesLogicalFrame) { // enabled above. TEST_F(TRTDecoderTest, CudaDeviceId_OutOfRangeThrows) { - if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; + if (!gpu_available()) + GTEST_SKIP() << "No CUDA GPU available"; std::string onnx_path = get_onnx_asset_path(); if (!std::filesystem::exists(onnx_path)) GTEST_SKIP() << "ONNX model not found: " << onnx_path; @@ -758,10 +758,12 @@ TEST_F(TRTDecoderTest, CudaDeviceId_OutOfRangeThrows) { } TEST_F(TRTDecoderTest, CudaDeviceId_DecodeAsyncOnGpu1) { - if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; + if (!gpu_available()) + GTEST_SKIP() << "No CUDA GPU available"; int count = 0; cudaGetDeviceCount(&count); - if (count < 2) GTEST_SKIP() << "needs >= 2 GPUs to prove non-default pinning"; + if (count < 2) + GTEST_SKIP() << "needs >= 2 GPUs to prove non-default pinning"; std::string onnx_path = get_onnx_asset_path(); if (!std::filesystem::exists(onnx_path)) GTEST_SKIP() << "ONNX model not found: " << onnx_path; @@ -778,7 +780,8 @@ TEST_F(TRTDecoderTest, CudaDeviceId_DecodeAsyncOnGpu1) { params.insert("cuda_device_id", 1); // engine/buffers must land on GPU 1 std::unique_ptr d; - ASSERT_NO_THROW({ d = cudaq::qec::decoder::get("trt_decoder", H_mat, params); }); + ASSERT_NO_THROW( + { d = cudaq::qec::decoder::get("trt_decoder", H_mat, params); }); // decode_async spawns a fresh thread (defaults to GPU 0). If trt did NOT // re-assert the device, the decode would run on GPU 0 while the engine lives @@ -792,58 +795,6 @@ TEST_F(TRTDecoderTest, CudaDeviceId_DecodeAsyncOnGpu1) { float trt_output = res.result[0]; float expected_output = TEST_OUTPUTS[0][0]; float error = std::abs(trt_output - expected_output); - EXPECT_LT(error, 1e-4f) - << "GPU-1 decode differs from expected: got " << trt_output - << ", expected " << expected_output; -} - -TEST(ScopedCudaDevice, NegativeIsNoop) { - if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; - int before = -1; - cudaGetDevice(&before); - { cudaq::qec::ScopedCudaDevice guard(-1); } - int after = -1; - cudaGetDevice(&after); - EXPECT_EQ(before, after); -} - -TEST(ScopedCudaDevice, SetsAndRestores) { - if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; - int count = 0; - cudaGetDeviceCount(&count); - if (count < 2) GTEST_SKIP() << "needs >= 2 GPUs"; - cudaSetDevice(0); - { - cudaq::qec::ScopedCudaDevice guard(1); - int cur = -1; - cudaGetDevice(&cur); - EXPECT_EQ(cur, 1); - } - int restored = -1; - cudaGetDevice(&restored); - EXPECT_EQ(restored, 0); -} - -TEST(ScopedCudaDevice, AllocLandsOnTarget) { - if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; - int count = 0; - cudaGetDeviceCount(&count); - if (count < 2) GTEST_SKIP() << "needs >= 2 GPUs"; - cudaSetDevice(0); - void *p = nullptr; - { - cudaq::qec::ScopedCudaDevice guard(1); - ASSERT_EQ(cudaMalloc(&p, 1024), cudaSuccess); - } - cudaPointerAttributes attr{}; - ASSERT_EQ(cudaPointerGetAttributes(&attr, p), cudaSuccess); - EXPECT_EQ(attr.device, 1); - cudaFree(p); -} - -TEST(ScopedCudaDevice, OutOfRangeThrows) { - if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; - int count = 0; - cudaGetDeviceCount(&count); - EXPECT_THROW({ cudaq::qec::ScopedCudaDevice guard(count); }, std::runtime_error); + EXPECT_LT(error, 1e-4f) << "GPU-1 decode differs from expected: got " + << trt_output << ", expected " << expected_output; } diff --git a/libs/qec/unittests/test_decoders.cpp b/libs/qec/unittests/test_decoders.cpp index d063bf670..e4011bf7a 100644 --- a/libs/qec/unittests/test_decoders.cpp +++ b/libs/qec/unittests/test_decoders.cpp @@ -12,6 +12,7 @@ #include "cudaq/qec/pcm_utils.h" #include #include +#include #include #include #include @@ -974,6 +975,33 @@ TEST(CudaDeviceId, OutOfRangeThrowsForAnyDecoder) { std::runtime_error); } +TEST(CudaDeviceId, DecodeAsyncRestoresCallerDevice) { + // Verify decoder::get() does not corrupt the calling thread's current device. + int count = 0; + cudaGetDeviceCount(&count); + if (count < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; + + cudaSetDevice(0); // caller stays on device 0 + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + params.insert("cuda_device_id", 1); // decoder assigned to device 1 + + // multi_error_lut is a CPU decoder -- it does nothing on device 1 itself, + // but the base class guard must still set device 1 in the async thread + // and then restore device 0 on the calling thread after get(). + auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); + + std::vector syn = {0.1f, 0.1f}; + auto fut = d->decode_async(syn); + fut.get(); // wait for completion + + int restored = -1; + cudaGetDevice(&restored); + EXPECT_EQ(restored, 0) + << "calling thread device not restored after decode_async"; +} + TEST(StimDemGetDecoder, ThrowsOnMalformedStimDem) { EXPECT_THROW(cudaq::qec::get_decoder("single_error_lut", "not a valid DEM"), std::runtime_error); diff --git a/libs/qec/unittests/test_device_affinity.cpp b/libs/qec/unittests/test_device_affinity.cpp index 267b8457e..11aae4395 100644 --- a/libs/qec/unittests/test_device_affinity.cpp +++ b/libs/qec/unittests/test_device_affinity.cpp @@ -9,13 +9,6 @@ #include "cudaq/qec/device_affinity.h" #include -#if defined(__linux__) -#include -#include -#include -#include -#endif - using cudaq::qec::read_cuda_device_id; using cudaq::qec::read_numa_node_id; using cudaqx::heterogeneous_map; @@ -34,7 +27,8 @@ TEST(DeviceAffinity, ReadIntStorage) { // YAML path: std::optional -> int EXPECT_EQ(read_numa_node_id(m), 1); } -TEST(DeviceAffinity, ReadSizeTStorage) { // kwargs path: Python int -> std::size_t +TEST(DeviceAffinity, + ReadSizeTStorage) { // kwargs path: Python int -> std::size_t heterogeneous_map m; m.insert("cuda_device_id", std::size_t{2}); EXPECT_EQ(read_cuda_device_id(m), 2); @@ -45,37 +39,3 @@ TEST(DeviceAffinity, ReadNegativeThrows) { m.insert("cuda_device_id", -1); EXPECT_THROW(read_cuda_device_id(m), std::runtime_error); } - -TEST(DeviceAffinity, ScopedNumaNodeNegativeIsNoop) { - EXPECT_NO_THROW({ cudaq::qec::ScopedNumaNode guard(-1); }); -} - -TEST(DeviceAffinity, ScopedNumaNodeZeroBinds) { - EXPECT_NO_THROW({ cudaq::qec::ScopedNumaNode guard(0); }); -} - -#if defined(__linux__) -TEST(DeviceAffinity, ScopedNumaNodeSetsAndRestoresMempolicy) { - int mode = -12345; - { - cudaq::qec::ScopedNumaNode guard(0); - long rc = syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL); - ASSERT_EQ(rc, 0); - EXPECT_EQ(mode, MPOL_BIND); - } - long rc = syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL); - ASSERT_EQ(rc, 0); - EXPECT_EQ(mode, MPOL_DEFAULT); -} - -TEST(DeviceAffinity, ScopedNumaNodeRestoresAffinity) { - cpu_set_t before; - CPU_ZERO(&before); - ASSERT_EQ(sched_getaffinity(0, sizeof(before), &before), 0); - { cudaq::qec::ScopedNumaNode guard(0); } - cpu_set_t after; - CPU_ZERO(&after); - ASSERT_EQ(sched_getaffinity(0, sizeof(after), &after), 0); - EXPECT_TRUE(CPU_EQUAL(&before, &after)); -} -#endif From 628c10b3d1ee2c381936642d835d837bfc9a02b7 Mon Sep 17 00:00:00 2001 From: kvmto Date: Wed, 1 Jul 2026 14:28:45 +0000 Subject: [PATCH 03/16] feat(qec): hardware-affinity knobs + loud, cudaq-decoupled pinning for decoders Add cuda_device_id/numa_node_id/cpu_affinity/mempolicy/pin_host_memory affinity controls to the base decoder, with safe-by-default posture (soft MPOL_PREFERRED, NUMA derived from the GPU, degrade-if-unknown). Pin via a persistent per-thread mechanism (bind_current_thread) so closed GPU decoders work without ABI changes; realtime session pins its dispatch thread and migrates ring buffers. Errors/violations are loud (throw on malformed/OOB, warn on OS-declined, info on auto-derive misses), and the affinity layer (hardware_affinity.h) is decoupled from the cudaq logger (plain-C diagnostics) so it is reusable standalone. Signed-off-by: kvmto --- libs/qec/include/cudaq/qec/decoder.h | 21 ++ libs/qec/include/cudaq/qec/device_affinity.h | 27 +++ libs/qec/lib/decoder.cpp | 197 +++++++++++------ libs/qec/lib/hardware_affinity.h | 205 ++++++++++++++++++ .../qec/lib/realtime/qec_realtime_session.cpp | 50 ++++- libs/qec/lib/realtime/qec_realtime_session.h | 3 + libs/qec/unittests/CMakeLists.txt | 5 +- .../qec/unittests/decoders/sample_decoder.cpp | 41 ++++ libs/qec/unittests/test_decoders.cpp | 192 ++++++++++++++++ libs/qec/unittests/test_device_affinity.cpp | 94 ++++++++ 10 files changed, 768 insertions(+), 67 deletions(-) create mode 100644 libs/qec/lib/hardware_affinity.h diff --git a/libs/qec/include/cudaq/qec/decoder.h b/libs/qec/include/cudaq/qec/decoder.h index b6ee6dff3..5be33b7e6 100644 --- a/libs/qec/include/cudaq/qec/decoder.h +++ b/libs/qec/include/cudaq/qec/decoder.h @@ -13,6 +13,7 @@ #include "cuda-qx/core/tensor.h" #include "sparse_binary_matrix.h" #include "cudaq/qec/detector_error_model.h" +#include "cudaq/qec/device_affinity.h" #include #include #include @@ -250,6 +251,19 @@ class decoder /// Not intended for direct use by plugin authors. void set_hardware_params(const cudaqx::heterogeneous_map ¶ms); + /// @brief Target NUMA node for this decoder (-1 = no binding). + int numa_node_id() const { return numa_node_id_; } + + /// @brief Target CUDA device for this decoder (-1 = inherit caller's device). + int cuda_device_id() const { return cuda_device_id_; } + + /// @brief Persistently bind the CALLING thread to this decoder's NUMA node + /// (and CUDA device). Call once from the thread that will own decode()/ + /// enqueue for this decoder (e.g. a realtime worker). No restore. + /// @return the node bound to, or -1 if nothing was applied. Warns if a node + /// was requested (>= 0) but could not be honored. + int bind_current_thread(); + // -- Begin realtime decoding API -- // Note: all of the current realtime decoding API is designed to be used with @@ -366,6 +380,13 @@ class decoder int cuda_device_id_ = -1; /// Target NUMA node for this decoder. -1 = no binding. int numa_node_id_ = -1; + /// Set once bind_current_thread() has pinned the owning thread, so the + /// synchronous decode_batch() guard can skip its redundant set/restore. + bool bound_persistently_ = false; + /// NUMA memory-policy mode for this decoder (default soft PREFERRED). + cudaq::qec::mempolicy_mode mempolicy_ = cudaq::qec::mempolicy_mode::preferred; + /// Explicit CPU cores for this decoder's owning thread (empty = use node cpuset). + std::vector cpu_affinity_; private: decode_result_type result_type_ = decode_result_type::decode_to_errs; diff --git a/libs/qec/include/cudaq/qec/device_affinity.h b/libs/qec/include/cudaq/qec/device_affinity.h index b911ded89..780a4ec32 100644 --- a/libs/qec/include/cudaq/qec/device_affinity.h +++ b/libs/qec/include/cudaq/qec/device_affinity.h @@ -11,6 +11,7 @@ #include "cuda-qx/core/heterogeneous_map.h" #include #include +#include namespace cudaq::qec { @@ -39,4 +40,30 @@ inline int read_numa_node_id(const cudaqx::heterogeneous_map ¶ms) { return detail::read_pin_key(params, "numa_node_id"); } +/// @brief Public NUMA memory-policy selector. preferred = soft; bind = strict. +enum class mempolicy_mode { preferred, bind }; + +/// @brief Read "mempolicy": "bind"->bind, "preferred"/absent->preferred, else throw. +inline mempolicy_mode read_mempolicy(const cudaqx::heterogeneous_map ¶ms) { + if (!params.contains("mempolicy")) + return mempolicy_mode::preferred; + const std::string v = params.get("mempolicy"); + if (v == "bind") + return mempolicy_mode::bind; + if (v == "preferred") + return mempolicy_mode::preferred; + throw std::runtime_error("mempolicy must be \"preferred\" or \"bind\" (got \"" + v + "\")"); +} + +/// @brief Read "cpu_affinity": a list of CPU core ids. Absent -> empty (no override). +inline std::vector read_cpu_affinity(const cudaqx::heterogeneous_map ¶ms) { + if (!params.contains("cpu_affinity")) + return {}; + return params.get>("cpu_affinity"); +} + +/// @brief NUMA node local to a CUDA device (via PCIe locality), or -1 if the +/// device id is negative or the topology can't be resolved. Defined in decoder.cpp. +int numa_node_for_cuda_device(int cuda_device_id); + } // namespace cudaq::qec diff --git a/libs/qec/lib/decoder.cpp b/libs/qec/lib/decoder.cpp index 6ffe9e856..86432a7a3 100644 --- a/libs/qec/lib/decoder.cpp +++ b/libs/qec/lib/decoder.cpp @@ -14,11 +14,14 @@ #include "cudaq/qec/version.h" #include "cudaq/runtime/logger/logger.h" #include +#include +#include #include #include #include #include #if defined(__linux__) +#include #include #include #include @@ -26,9 +29,17 @@ #include #include #endif +#include "hardware_affinity.h" namespace { +inline cudaq::qec::detail_affinity::mempolicy_mode +to_affinity_mode(cudaq::qec::mempolicy_mode m) { + return m == cudaq::qec::mempolicy_mode::bind + ? cudaq::qec::detail_affinity::mempolicy_mode::bind + : cudaq::qec::detail_affinity::mempolicy_mode::preferred; +} + // RAII: sets the calling thread's CUDA current device, restores on destruction. // target < 0 = no-op. Throws std::runtime_error if target is out of range. struct CudaDeviceGuard { @@ -43,9 +54,12 @@ struct CudaDeviceGuard { throw std::runtime_error( "cuda_device_id " + std::to_string(target) + " out of range (device_count=" + std::to_string(count) + ")"); - cudaGetDevice(&prev_); + if (cudaGetDevice(&prev_) != cudaSuccess) { prev_ = -1; return; } // can't safely switch if (prev_ != target) { - cudaSetDevice(target); + cudaError_t e = cudaSetDevice(target); + if (e != cudaSuccess) + throw std::runtime_error("CudaDeviceGuard: cudaSetDevice(" + std::to_string(target) + + ") failed: " + cudaGetErrorString(e)); active_ = true; } } @@ -58,73 +72,38 @@ struct CudaDeviceGuard { }; #if defined(__linux__) -// Parse /sys/devices/system/node/node/cpulist e.g. "0-7,16-23". -// Returns false if the file is missing or empty. -static bool build_node_cpuset(int node, cpu_set_t &out) { - std::ifstream f("/sys/devices/system/node/node" + std::to_string(node) + - "/cpulist"); - if (!f.is_open()) - return false; - std::string list; - std::getline(f, list); - if (list.empty()) - return false; - std::stringstream ss(list); - std::string range; - bool any = false; - while (std::getline(ss, range, ',')) { - auto dash = range.find('-'); - int lo = std::stoi(range.substr(0, dash)); - int hi = - (dash == std::string::npos) ? lo : std::stoi(range.substr(dash + 1)); - for (int c = lo; c <= hi; ++c) { - if (c < CPU_SETSIZE) { // guard against OOB on >1024-CPU machines - CPU_SET(c, &out); - any = true; - } - } - } - return any; -} - -// RAII: binds the calling thread's CPU affinity and memory policy to a NUMA -// node, restores on destruction. node < 0 = no-op. -// Uses raw Linux syscalls -- no libnuma dependency. +// RAII: binds the calling thread to a NUMA node and restores on destruction. +// node < 0 = no-op. Uses the shared persistent-bind primitive for the set half +// and remembers prior affinity for the restore. struct NumaGuard { - bool mempol_set_ = false; bool affinity_set_ = false; bool has_prev_affinity_ = false; + bool mempol_set_ = false; cpu_set_t prev_set_{}; - explicit NumaGuard(int node) { + explicit NumaGuard(int node, cudaq::qec::detail_affinity::mempolicy_mode mode = + cudaq::qec::detail_affinity::mempolicy_mode::preferred) { if (node < 0) return; CPU_ZERO(&prev_set_); - if (sched_getaffinity(0, sizeof(prev_set_), &prev_set_) == 0) - has_prev_affinity_ = true; - - cpu_set_t node_set; - CPU_ZERO(&node_set); - // Only pin CPU affinity when we can restore it; avoids permanent pinning - // if sched_getaffinity failed (e.g., container with locked cpuset). - if (has_prev_affinity_ && build_node_cpuset(node, node_set)) { - if (sched_setaffinity(0, sizeof(node_set), &node_set) == 0) - affinity_set_ = true; - } - - if (node < static_cast(sizeof(unsigned long) * 8)) { - unsigned long nodemask = 1UL << node; - if (syscall(SYS_set_mempolicy, MPOL_BIND, &nodemask, - static_cast(sizeof(nodemask) * 8)) == 0) - mempol_set_ = true; - } + has_prev_affinity_ = + (sched_getaffinity(0, sizeof(prev_set_), &prev_set_) == 0); + affinity_set_ = has_prev_affinity_; + mempol_set_ = (node < static_cast(sizeof(unsigned long) * 8)); + cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(node, mode); } ~NumaGuard() { if (mempol_set_) - syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); - if (affinity_set_) - sched_setaffinity(0, sizeof(prev_set_), &prev_set_); + if (syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL) != 0) + cudaq::qec::detail_affinity::affinity_warn( + "NumaGuard restore: failed to reset thread mempolicy: " + + std::string(std::strerror(errno)) + "; thread may remain bound"); + if (affinity_set_ && has_prev_affinity_) + if (sched_setaffinity(0, sizeof(prev_set_), &prev_set_) != 0) + cudaq::qec::detail_affinity::affinity_warn( + "NumaGuard restore: failed to reset thread affinity: " + + std::string(std::strerror(errno)) + "; thread may remain pinned"); } NumaGuard(const NumaGuard &) = delete; @@ -132,7 +111,9 @@ struct NumaGuard { }; #else struct NumaGuard { - explicit NumaGuard(int node) { + explicit NumaGuard(int node, cudaq::qec::detail_affinity::mempolicy_mode mode = + cudaq::qec::detail_affinity::mempolicy_mode::preferred) { + (void)mode; if (node < 0) return; static bool warned = false; @@ -217,9 +198,88 @@ decoder::decoder(cudaq::qec::sparse_binary_matrix H) pimpl->should_log = ch[0] == '1' || ch[0] == 'y' || ch[0] == 'Y'; } +int numa_node_for_cuda_device(int cuda_device_id) { + if (cuda_device_id < 0) + return -1; +#if defined(__linux__) + char busid[32] = {0}; + if (cudaDeviceGetPCIBusId(busid, sizeof(busid), cuda_device_id) != cudaSuccess) { + cudaq::qec::detail_affinity::affinity_info( + "numa_node_id auto-derive for cuda_device_id " + + std::to_string(cuda_device_id) + + " could not read the PCI bus id; NUMA binding skipped"); + return -1; + } + for (char *c = busid; *c; ++c) + *c = static_cast(std::tolower(static_cast(*c))); + std::ifstream f(std::string("/sys/bus/pci/devices/") + busid + "/numa_node"); + int node = -1; + if (!f.is_open() || !(f >> node) || node < 0) { + cudaq::qec::detail_affinity::affinity_info( + "numa_node_id auto-derive for cuda_device_id " + + std::to_string(cuda_device_id) + + " resolved to no node (single-node host or unresolved topology); " + "NUMA binding skipped"); + return -1; + } + return node; +#else + return -1; +#endif +} + void decoder::set_hardware_params(const cudaqx::heterogeneous_map ¶ms) { cuda_device_id_ = cudaq::qec::read_cuda_device_id(params); numa_node_id_ = cudaq::qec::read_numa_node_id(params); + mempolicy_ = cudaq::qec::read_mempolicy(params); + cpu_affinity_ = cudaq::qec::read_cpu_affinity(params); + // Soft-derive: user pinned a GPU but not a node -> use the GPU's node (or -1 if unresolved). + if (numa_node_id_ < 0 && cuda_device_id_ >= 0) + numa_node_id_ = numa_node_for_cuda_device(cuda_device_id_); +} + +int decoder::bind_current_thread() { + if (cuda_device_id_ >= 0) { + // Persistent set on this thread (CudaDeviceGuard restores on scope exit, so + // set directly here and let it stick). + int count = 0; + if (cudaGetDeviceCount(&count) != cudaSuccess || cuda_device_id_ >= count) + throw std::runtime_error("cuda_device_id " + std::to_string(cuda_device_id_) + + " out of range or CUDA unavailable in bind_current_thread"); + cudaError_t e = cudaSetDevice(cuda_device_id_); + if (e != cudaSuccess) + throw std::runtime_error("bind_current_thread: cudaSetDevice(" + + std::to_string(cuda_device_id_) + ") failed: " + cudaGetErrorString(e)); + } + cudaq::qec::detail_affinity::bind_this_thread_to_numa_node( + numa_node_id_, to_affinity_mode(mempolicy_)); + bound_persistently_ = true; + if (numa_node_id_ >= 0) { + // Best-effort confirmation: warn if the calling thread is not actually + // running on the requested node's CPUs (e.g. locked cpuset in a container). +#if defined(__linux__) + cpu_set_t want, have; + CPU_ZERO(&want); + CPU_ZERO(&have); + if (cudaq::qec::detail_affinity::build_node_cpuset(numa_node_id_, want) && + sched_getaffinity(0, sizeof(have), &have) == 0) { + bool on_node = false; + for (int c = 0; c < CPU_SETSIZE; ++c) + if (CPU_ISSET(c, &have) && CPU_ISSET(c, &want)) { + on_node = true; + break; + } + if (!on_node) + cudaq::qec::detail_affinity::affinity_warn( + "bind_current_thread: numa_node_id " + + std::to_string(numa_node_id_) + + " requested but the calling thread could not be pinned to it"); + } +#endif + } + if (!cpu_affinity_.empty()) + cudaq::qec::detail_affinity::set_thread_cpu_affinity(cpu_affinity_); + return numa_node_id_; } // Provide a trivial implementation of for tensor decode call. Child @@ -244,8 +304,11 @@ std::vector decoder::decode_batch(const std::vector> &syndrome) { // Apply affinity once for the whole batch (not per-syndrome) to avoid // repeated syscall overhead on the hot path. - CudaDeviceGuard dev(cuda_device_id_); - NumaGuard numa(numa_node_id_); + // If the caller already bound this thread via bind_current_thread(), the + // per-call guards are redundant set/restore syscalls — skip them. + CudaDeviceGuard dev(bound_persistently_ ? -1 : cuda_device_id_); + NumaGuard numa(bound_persistently_ ? -1 : numa_node_id_, + to_affinity_mode(mempolicy_)); std::vector result; result.reserve(syndrome.size()); for (auto &s : syndrome) @@ -266,9 +329,11 @@ decoder::decode_async(const std::vector &syndrome) { // the future resolves. const int cuda_id = cuda_device_id_; const int numa_id = numa_node_id_; - return std::async(std::launch::async, [this, syndrome, cuda_id, numa_id] { + const cudaq::qec::mempolicy_mode mempolicy = mempolicy_; + return std::async(std::launch::async, [this, syndrome, cuda_id, numa_id, + mempolicy] { CudaDeviceGuard dev(cuda_id); - NumaGuard numa(numa_id); + NumaGuard numa(numa_id, to_affinity_mode(mempolicy)); return this->decode(syndrome); }); } @@ -287,8 +352,12 @@ decoder::get(const std::string &name, const decoder_init &init, // Guards during construction so allocations land on the right hardware. // Restored before this function returns; decode-time affinity is re-applied // per call in decode_batch() and decode_async(). - CudaDeviceGuard ctor_dev(cudaq::qec::read_cuda_device_id(param_map)); - NumaGuard ctor_numa(cudaq::qec::read_numa_node_id(param_map)); + const int dev = cudaq::qec::read_cuda_device_id(param_map); + int node = cudaq::qec::read_numa_node_id(param_map); + if (node < 0 && dev >= 0) + node = numa_node_for_cuda_device(dev); + CudaDeviceGuard ctor_dev(dev); + NumaGuard ctor_numa(node); // preferred mode at construction auto d = iter->second(init, param_map); d->set_hardware_params(param_map); return d; diff --git a/libs/qec/lib/hardware_affinity.h b/libs/qec/lib/hardware_affinity.h new file mode 100644 index 000000000..81665a0d3 --- /dev/null +++ b/libs/qec/lib/hardware_affinity.h @@ -0,0 +1,205 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ + +// Lib-private (NOT installed): hardware affinity primitives shared by the base +// decoder and the realtime session. Never include from a public header. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +namespace cudaq::qec::detail_affinity { + +// Decoupled diagnostics: plain C, no cudaq dependency, so this header can be reused independently. +inline void affinity_warn(const std::string &msg) { + std::fprintf(stderr, "[cudaq-qec affinity] WARNING: %s\n", msg.c_str()); +} +// INFO/debug: only shown when CUDAQ_QEC_AFFINITY_DEBUG is set (a legitimate no-op like a +// single-node auto-derive must not spam; it is debug-loggable, not a failure). +inline void affinity_info(const std::string &msg) { + if (std::getenv("CUDAQ_QEC_AFFINITY_DEBUG") != nullptr) + std::fprintf(stderr, "[cudaq-qec affinity] INFO: %s\n", msg.c_str()); +} + +#if defined(__linux__) + +enum class mempolicy_mode { preferred, bind }; +inline int mempolicy_syscall_mode(mempolicy_mode m) { + return (m == mempolicy_mode::bind) ? MPOL_BIND : MPOL_PREFERRED; +} + +// Parse /sys/devices/system/node/node/cpulist e.g. "0-7,16-23". +// Returns false if the file is missing or empty. +inline bool build_node_cpuset(int node, cpu_set_t &out) { + std::ifstream f("/sys/devices/system/node/node" + std::to_string(node) + + "/cpulist"); + if (!f.is_open()) + return false; + std::string list; + std::getline(f, list); + if (list.empty()) + return false; + std::stringstream ss(list); + std::string range; + bool any = false; + while (std::getline(ss, range, ',')) { + auto dash = range.find('-'); + int lo = std::stoi(range.substr(0, dash)); + int hi = + (dash == std::string::npos) ? lo : std::stoi(range.substr(dash + 1)); + for (int c = lo; c <= hi; ++c) { + if (c < CPU_SETSIZE) { // guard against OOB on >1024-CPU machines + CPU_SET(c, &out); + any = true; + } + } + } + return any; +} + +// Persistently bind the CALLING thread's CPU affinity + memory policy to a NUMA +// node. No restore. node < 0 = no-op. Only pins CPU affinity when prior +// affinity is readable (avoids permanent pinning in a locked-cpuset container). +// Throws if node cannot be encoded in the mempolicy nodemask (node >= 64). +// Warns (does not throw) if the OS declines an otherwise well-formed request. +inline void bind_this_thread_to_numa_node(int node, + mempolicy_mode mode = mempolicy_mode::preferred) { + if (node < 0) + return; + if (node >= static_cast(sizeof(unsigned long) * 8)) + throw std::runtime_error( + "numa_node_id " + std::to_string(node) + + " exceeds the maximum encodable NUMA node (" + + std::to_string(sizeof(unsigned long) * 8 - 1) + "); cannot bind memory policy"); + cpu_set_t prev; + CPU_ZERO(&prev); + const bool can_restore = (sched_getaffinity(0, sizeof(prev), &prev) == 0); + cpu_set_t node_set; + CPU_ZERO(&node_set); + if (can_restore && build_node_cpuset(node, node_set)) { + if (sched_setaffinity(0, sizeof(node_set), &node_set) != 0) + affinity_warn("numa_node_id " + std::to_string(node) + + " requested but sched_setaffinity could not pin the thread: " + + std::strerror(errno) + "; running unpinned (locked cpuset?)"); + } else { + affinity_warn("numa_node_id " + std::to_string(node) + + " requested but its CPU list could not be read (or prior " + "affinity unreadable); CPU affinity left unpinned"); + } + unsigned long nodemask = 1UL << node; + if (syscall(SYS_set_mempolicy, mempolicy_syscall_mode(mode), &nodemask, + static_cast(sizeof(nodemask) * 8)) != 0) + affinity_warn("numa_node_id " + std::to_string(node) + + " requested but set_mempolicy failed: " + std::strerror(errno) + + "; memory not bound to node"); +} + +// Migrate an already-allocated region onto a NUMA node. The ring buffers are +// calloc'd (already faulted) on the setup thread, so MPOL_MF_MOVE is required +// to relocate the pages. node < 0 / null / 0 bytes = no-op. +// Throws if node cannot be encoded in the mempolicy nodemask (node >= 64). +// Warns (does not throw) if the OS declines an otherwise well-formed request. +inline void bind_region_to_numa_node(void *p, std::size_t bytes, int node, + mempolicy_mode mode = mempolicy_mode::preferred) { + if (node < 0 || p == nullptr || bytes == 0) + return; + if (node >= static_cast(sizeof(unsigned long) * 8)) + throw std::runtime_error( + "numa_node_id " + std::to_string(node) + + " exceeds the maximum encodable NUMA node; cannot migrate region"); + unsigned long nodemask = 1UL << node; + if (syscall(SYS_mbind, p, static_cast(bytes), + mempolicy_syscall_mode(mode), &nodemask, + static_cast(sizeof(nodemask) * 8), MPOL_MF_MOVE) != 0) + affinity_warn("mbind of region to numa_node_id " + std::to_string(node) + + " failed: " + std::strerror(errno) + + "; pages not migrated (missing CAP_SYS_NICE?)"); +} + +// Query the calling thread's current memory-policy mode (MPOL_* constant). +// Returns -1 if the query fails. +inline int current_thread_mempolicy_mode() { + int mode = -1; + if (syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL) != 0) + return -1; + return mode; +} + +// Pin the CALLING thread's CPU affinity to an explicit list of core ids. +// No restore. cores.empty() = no-op. Throws std::invalid_argument if any core +// id is out of range. Warns (does not throw) if the OS declines an otherwise +// well-formed request. +inline void set_thread_cpu_affinity(const std::vector &cores) { + if (cores.empty()) + return; + for (int c : cores) + if (c < 0 || c >= CPU_SETSIZE) + throw std::invalid_argument("cpu_affinity core id " + std::to_string(c) + + " is out of range [0, " + std::to_string(CPU_SETSIZE) + ")"); + cpu_set_t set; + CPU_ZERO(&set); + for (int c : cores) + CPU_SET(c, &set); + if (sched_setaffinity(0, sizeof(set), &set) != 0) + affinity_warn("cpu_affinity requested but sched_setaffinity failed: " + + std::string(std::strerror(errno)) + + "; thread left unpinned (locked cpuset?)"); +} + +// Query the calling thread's current CPU affinity set as a sorted list of core +// ids. Returns an empty vector if the query fails. +inline std::vector current_thread_cpuset() { + std::vector out; + cpu_set_t set; + CPU_ZERO(&set); + if (sched_getaffinity(0, sizeof(set), &set) != 0) + return out; + for (int c = 0; c < CPU_SETSIZE; ++c) + if (CPU_ISSET(c, &set)) + out.push_back(c); + return out; +} + +#else // non-Linux: no-ops + +enum class mempolicy_mode { preferred, bind }; +inline void bind_this_thread_to_numa_node(int node, + mempolicy_mode = mempolicy_mode::preferred) { + if (node < 0) + return; + static bool warned = false; + if (!warned) { + affinity_warn("numa_node_id ignored: NUMA binding is only supported on Linux"); + warned = true; + } +} +inline void bind_region_to_numa_node(void *, std::size_t, int, + mempolicy_mode = mempolicy_mode::preferred) {} +inline int current_thread_mempolicy_mode() { return -1; } +inline void set_thread_cpu_affinity(const std::vector &) {} +inline std::vector current_thread_cpuset() { return {}; } + +#endif + +} // namespace cudaq::qec::detail_affinity diff --git a/libs/qec/lib/realtime/qec_realtime_session.cpp b/libs/qec/lib/realtime/qec_realtime_session.cpp index 3fc943b1e..5245aec3c 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.cpp +++ b/libs/qec/lib/realtime/qec_realtime_session.cpp @@ -10,6 +10,7 @@ #include "qec_realtime_session.h" +#include "hardware_affinity.h" #include "cudaq/qec/realtime/decoder_rpc_ids.h" #include "cudaq/qec/realtime/graph_resources.h" #include "cudaq/realtime/daemon/dispatcher/dispatch_kernel_launch.h" @@ -328,6 +329,31 @@ void qec_realtime_session::initialize() { try { if (device_mode_) capture_decoder_graphs(); + // Resolve the session NUMA node from the decoders. A single shared dispatch + // thread can honor only one node; warn and disable pinning if they + // disagree. + { + int chosen = -1; + bool conflict = false; + for (const auto &d : decoders_) { + const int n = d->numa_node_id(); + if (n < 0) + continue; + if (chosen < 0) + chosen = n; + else if (chosen != n) + conflict = true; + } + if (conflict) { + CUDAQ_WARN( + "realtime decoders request different numa_node_ids; the shared " + "host dispatch thread can honor only one. NUMA pinning " + "disabled for this session."); + session_numa_node_ = -1; + } else { + session_numa_node_ = chosen; + } + } allocate_ring_buffer(); populate_function_table(); if (device_mode_) @@ -619,6 +645,22 @@ void qec_realtime_session::allocate_ring_buffer() { alloc_u64(tx_flags_host_, tx_flags_dev_, "tx_flags"); alloc_u8(rx_data_host_, rx_data_dev_, "RX ring data"); alloc_u8(tx_data_host_, tx_data_dev_, "TX ring data"); + // Migrate the syndrome ring buffers onto the session node so the (pinned) + // dispatch thread reads them on-node. calloc already faulted these on the + // setup thread, so this relocates the pages (MPOL_MF_MOVE). + if (session_numa_node_ >= 0) { + using cudaq::qec::detail_affinity::bind_region_to_numa_node; + bind_region_to_numa_node((void *)rx_flags_host_, + num_slots_ * sizeof(std::uint64_t), + session_numa_node_); + bind_region_to_numa_node((void *)tx_flags_host_, + num_slots_ * sizeof(std::uint64_t), + session_numa_node_); + bind_region_to_numa_node((void *)rx_data_host_, num_slots_ * slot_size_, + session_numa_node_); + bind_region_to_numa_node((void *)tx_data_host_, num_slots_ * slot_size_, + session_numa_node_); + } } std::memset(&ringbuffer_, 0, sizeof(ringbuffer_)); @@ -843,8 +885,12 @@ void qec_realtime_session::start_host_loop() { host_ctx_.skip_stream_sweep = true; shutdown_flag_ = 0; - host_loop_thread_ = - std::thread([this]() { cudaq_host_dispatcher_loop(&host_ctx_); }); + const int node = session_numa_node_; + host_loop_thread_ = std::thread([this, node]() { + // Persistent bind: this thread does nothing but decode for its lifetime. + cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(node); + cudaq_host_dispatcher_loop(&host_ctx_); + }); return; } diff --git a/libs/qec/lib/realtime/qec_realtime_session.h b/libs/qec/lib/realtime/qec_realtime_session.h index 833aa8805..62661e47b 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.h +++ b/libs/qec/lib/realtime/qec_realtime_session.h @@ -208,6 +208,9 @@ class __attribute__((visibility("default"))) qec_realtime_session { // ---- HOST_LOOP wiring (both modes) ---- cudaq_host_dispatch_loop_ctx_t host_ctx_{}; std::thread host_loop_thread_; + /// Single NUMA node this session pins to (dispatch thread + ring buffers). + /// -1 = no pinning (unset, or decoders disagree — see initialize()). + int session_numa_node_ = -1; std::uint64_t host_stats_counter_ = 0; // Plain (non-pinned) shutdown flag for HOST mode (no device kernel shares // it). diff --git a/libs/qec/unittests/CMakeLists.txt b/libs/qec/unittests/CMakeLists.txt index 1435c1bf4..173e3e76a 100644 --- a/libs/qec/unittests/CMakeLists.txt +++ b/libs/qec/unittests/CMakeLists.txt @@ -51,7 +51,10 @@ add_dependencies(CUDAQXQECUnitTests test_decoders_yaml) gtest_discover_tests(test_decoders_yaml) add_executable(test_device_affinity test_device_affinity.cpp) -target_link_libraries(test_device_affinity PRIVATE GTest::gtest_main cudaq-qec cudaq::cudaq) +target_link_libraries(test_device_affinity PRIVATE GTest::gtest_main cudaq-qec cudaq::cudaq CUDA::cudart) +target_include_directories(test_device_affinity PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../lib + ${CUDAToolkit_INCLUDE_DIRS}) add_dependencies(CUDAQXQECUnitTests test_device_affinity) gtest_discover_tests(test_device_affinity) diff --git a/libs/qec/unittests/decoders/sample_decoder.cpp b/libs/qec/unittests/decoders/sample_decoder.cpp index 0d357b5d7..bbc4c52f9 100644 --- a/libs/qec/unittests/decoders/sample_decoder.cpp +++ b/libs/qec/unittests/decoders/sample_decoder.cpp @@ -7,6 +7,7 @@ ******************************************************************************/ #include "cudaq/qec/decoder.h" +#include #include using namespace cudaqx; @@ -50,4 +51,44 @@ class sample_decoder : public decoder { CUDAQ_EXT_PT_REGISTER_TYPE(sample_decoder) +/// @brief Test-only decoder shaped like a typical GPU decoder: it overrides +/// ONLY the single-syndrome decode() (NOT decode_batch) and "allocates lazily" +/// on first decode(). It reports, via result.result[0], the CUDA device its +/// allocation actually landed on — so a test can assert cuda_device_id was +/// honored on whichever entry point invoked it. +class device_probe_decoder : public decoder { +public: + device_probe_decoder(const cudaq::qec::sparse_binary_matrix &H, + const cudaqx::heterogeneous_map ¶ms) + : decoder(H) {} + + virtual decoder_result decode(const std::vector &syndrome) override { + decoder_result result; + result.converged = true; + int dev = -1; + void *p = nullptr; + if (cudaMalloc(&p, 16) == cudaSuccess && p) { + cudaPointerAttributes attr{}; + if (cudaPointerGetAttributes(&attr, p) == cudaSuccess) + dev = attr.device; // device the lazy allocation landed on + cudaFree(p); + } else { + cudaGetDevice(&dev); + } + result.result = std::vector{static_cast(dev)}; + return result; + } + + virtual ~device_probe_decoder() {} + + CUDAQ_EXTENSION_CUSTOM_CREATOR_FUNCTION( + device_probe_decoder, static std::unique_ptr create( + const cudaq::qec::decoder_init &init, + const cudaqx::heterogeneous_map ¶ms) { + return cudaq::qec::make_pcm_decoder(init, params); + }) +}; + +CUDAQ_EXT_PT_REGISTER_TYPE(device_probe_decoder) + } // namespace cudaq::qec diff --git a/libs/qec/unittests/test_decoders.cpp b/libs/qec/unittests/test_decoders.cpp index e4011bf7a..1404e0257 100644 --- a/libs/qec/unittests/test_decoders.cpp +++ b/libs/qec/unittests/test_decoders.cpp @@ -17,6 +17,10 @@ #include #include #include +#include +#if defined(__linux__) +#include +#endif namespace { class ScopedEnv { @@ -965,6 +969,32 @@ TEST(StimDemGetDecoder, ThrowsOnProbabilityOutOfRange) { std::runtime_error); } +TEST(HardwarePinning, AccessorsReturnStoredValues) { + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + params.insert("numa_node_id", 0); + auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); + EXPECT_EQ(d->numa_node_id(), 0); + EXPECT_EQ(d->cuda_device_id(), -1); // unset +} + +TEST(HardwarePinning, BindCurrentThreadNoopWhenUnset) { + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; // no numa_node_id + auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); + EXPECT_EQ(d->bind_current_thread(), -1); // nothing to bind +} + +TEST(HardwarePinning, DecodeStillWorksAfterBind) { + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); + d->bind_current_thread(); // no-op at -1, but must not break decode + std::vector> batch = {{0.1f, 0.1f}}; + auto results = d->decode_batch(batch); + EXPECT_EQ(results.size(), 1u); +} + TEST(CudaDeviceId, OutOfRangeThrowsForAnyDecoder) { // A device id beyond the available devices is rejected during construction // for any decoder, including CPU ones. @@ -1002,6 +1032,64 @@ TEST(CudaDeviceId, DecodeAsyncRestoresCallerDevice) { << "calling thread device not restored after decode_async"; } +// On a multi-GPU box: after binding the caller to device 1, decode_batch must +// NOT restore the caller to device 0 (it should leave the persistent binding +// in place). Skipped where < 2 GPUs. +TEST(HardwarePinning, DecodeBatchLeavesPersistentDeviceBinding) { + int count = 0; + cudaGetDeviceCount(&count); + if (count < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; + + cudaSetDevice(0); + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + params.insert("cuda_device_id", 1); + auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); + + d->bind_current_thread(); // caller now persistently on device 1 + std::vector> batch = {{0.1f, 0.1f}}; + (void)d->decode_batch(batch); + + int dev = -1; + cudaGetDevice(&dev); + EXPECT_EQ(dev, 1) << "decode_batch restored device despite persistent bind"; +} + +TEST(HardwarePinning, BindCurrentThreadAppliesCpuAffinity) { + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + params.insert("cpu_affinity", std::vector{0}); + auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); + std::thread t([&] { + d->bind_current_thread(); +#if defined(__linux__) + cpu_set_t have; CPU_ZERO(&have); sched_getaffinity(0, sizeof(have), &have); + EXPECT_TRUE(CPU_ISSET(0, &have)); + EXPECT_EQ(CPU_COUNT(&have), 1); +#endif + }); + t.join(); +} + +// NOTE: construction (decoder::get) already runs CudaDeviceGuard ctor_dev(dev) +// before the decoder object is created/returned, so an out-of-range +// cuda_device_id throws AT CONSTRUCTION, not inside bind_current_thread(). +// bind_current_thread() is hardened the same way (see decoder.cpp) for the +// case where a decoder is constructed with cuda_device_id unset/valid and a +// caller later attempts to bind to an out-of-range device directly, but this +// test's OOB device is supplied via decoder::get()'s param_map, so the throw +// fires during get() itself. Assert the throw at the layer where it actually +// fires: construction. +TEST(HardwarePinning, BindCurrentThreadThrowsOnOutOfRangeDevice) { + int n = 0; cudaGetDeviceCount(&n); + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + params.insert("cuda_device_id", n + 100); // guaranteed OOB + EXPECT_THROW(cudaq::qec::decoder::get("multi_error_lut", H, params), + std::runtime_error); +} + TEST(StimDemGetDecoder, ThrowsOnMalformedStimDem) { EXPECT_THROW(cudaq::qec::get_decoder("single_error_lut", "not a valid DEM"), std::runtime_error); @@ -1169,3 +1257,107 @@ TEST(SlidingWindowDecoder, BaseStreamingCopiesFirstRoundDetectors) { << "First-round detector copy runs, but the sliding window is not full " "yet so no final correction is committed."; } + +// GPU placement: cuda_device_id must be honored across the decode entry points. +// A decoder that overrides only decode() and allocates lazily still lands on the +// assigned GPU when its owning thread is pinned via bind_current_thread(). + +TEST(HardwarePinningGpu, RawDecodeOnPinnedThreadUsesAssignedGpu) { + int n = 0; cudaGetDeviceCount(&n); + if (n < 2) GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map p; p.insert("cuda_device_id", 1); + auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); + int observed = -1; + std::size_t sz = 0; + std::thread worker([&] { + d->bind_current_thread(); // owns this decoder; pins current device to 1 + auto r = d->decode(std::vector{0.1f, 0.1f}); + sz = r.result.size(); + if (sz) + observed = static_cast(r.result[0]); + }); + worker.join(); + ASSERT_EQ(sz, 1u); + EXPECT_EQ(observed, 1) << "decode() on a pinned worker did not use the assigned GPU"; +} + +// Two logical patches on two GPUs: each decoder, pinned to its own GPU on its own +// worker thread, decodes on its assigned GPU with no cross-talk. +TEST(HardwarePinningGpu, MultiPatchOnPinnedThreadsUseDistinctGpus) { + int n = 0; cudaGetDeviceCount(&n); + if (n < 2) GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map p0; p0.insert("cuda_device_id", 0); + cudaqx::heterogeneous_map p1; p1.insert("cuda_device_id", 1); + auto d0 = cudaq::qec::decoder::get("device_probe_decoder", H, p0); + auto d1 = cudaq::qec::decoder::get("device_probe_decoder", H, p1); + int o0 = -1, o1 = -1; + std::thread t0([&] { + d0->bind_current_thread(); + auto r = d0->decode(std::vector{0.1f, 0.1f}); + if (r.result.size()) + o0 = static_cast(r.result[0]); + }); + std::thread t1([&] { + d1->bind_current_thread(); + auto r = d1->decode(std::vector{0.1f, 0.1f}); + if (r.result.size()) + o1 = static_cast(r.result[0]); + }); + t0.join(); + t1.join(); + EXPECT_EQ(o0, 0) << "patch 0 did not use GPU 0"; + EXPECT_EQ(o1, 1) << "patch 1 did not use GPU 1"; +} + +// decode_async applies the device guard on its worker, so it honors cuda_device_id. +TEST(HardwarePinningGpu, DecodeAsyncHonorsCudaDeviceId) { + int n = 0; cudaGetDeviceCount(&n); + if (n < 2) GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map p; p.insert("cuda_device_id", 1); + auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); + auto fut = d->decode_async(std::vector{0.1f, 0.1f}); + auto r = fut.get(); + EXPECT_EQ(static_cast(r.result[0]), 1) + << "decode_async did not honor cuda_device_id"; +} + +// decode_batch applies the device guard, so it honors cuda_device_id. +TEST(HardwarePinningGpu, DecodeBatchHonorsCudaDeviceId) { + int n = 0; cudaGetDeviceCount(&n); + if (n < 2) GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map p; p.insert("cuda_device_id", 1); + auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); + auto rs = d->decode_batch( + std::vector>{{0.1f, 0.1f}}); + ASSERT_EQ(rs.size(), 1u); + EXPECT_EQ(static_cast(rs[0].result[0]), 1) + << "decode_batch did not honor cuda_device_id"; +} + +TEST(HardwarePinning, NumaDerivesFromCudaDeviceOrDegrades) { + int n = 0; cudaGetDeviceCount(&n); + if (n < 1) GTEST_SKIP() << "needs a GPU"; + int node = cudaq::qec::numa_node_for_cuda_device(0); + EXPECT_GE(node, -1); // real node on multi-node HW, or -1 + EXPECT_EQ(cudaq::qec::numa_node_for_cuda_device(-1), -1); // negative device -> -1 +} +TEST(HardwarePinning, GetDecoderSoftDerivesNumaWhenOnlyCudaSet) { + int n = 0; cudaGetDeviceCount(&n); + if (n < 1) GTEST_SKIP() << "needs a GPU"; + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + params.insert("cuda_device_id", 0); // numa_node_id NOT set + auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); + EXPECT_EQ(d->numa_node_id(), cudaq::qec::numa_node_for_cuda_device(0)); // derivation ran +} +TEST(HardwarePinning, NumaAutoDeriveUnknownIsInfoNotThrow) { + EXPECT_NO_THROW({ EXPECT_EQ(cudaq::qec::numa_node_for_cuda_device(-1), -1); }); +} diff --git a/libs/qec/unittests/test_device_affinity.cpp b/libs/qec/unittests/test_device_affinity.cpp index 11aae4395..27a184d55 100644 --- a/libs/qec/unittests/test_device_affinity.cpp +++ b/libs/qec/unittests/test_device_affinity.cpp @@ -6,9 +6,19 @@ * the terms of the Apache License 2.0 which accompanies this distribution. * ******************************************************************************/ +#include "cuda-qx/core/heterogeneous_map.h" #include "cudaq/qec/device_affinity.h" +#include #include +#if defined(__linux__) +#include "hardware_affinity.h" +#include +#include +#include +#endif +#include + using cudaq::qec::read_cuda_device_id; using cudaq::qec::read_numa_node_id; using cudaqx::heterogeneous_map; @@ -39,3 +49,87 @@ TEST(DeviceAffinity, ReadNegativeThrows) { m.insert("cuda_device_id", -1); EXPECT_THROW(read_cuda_device_id(m), std::runtime_error); } + +#if defined(__linux__) +TEST(HardwareAffinity, MempolicyDefaultIsPreferredNotBind) { + namespace da = cudaq::qec::detail_affinity; + da::bind_this_thread_to_numa_node(0); + EXPECT_EQ(da::current_thread_mempolicy_mode(), MPOL_PREFERRED); + syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); +} +TEST(HardwareAffinity, MempolicyBindWhenRequested) { + namespace da = cudaq::qec::detail_affinity; + da::bind_this_thread_to_numa_node(0, da::mempolicy_mode::bind); + EXPECT_EQ(da::current_thread_mempolicy_mode(), MPOL_BIND); + syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); +} +TEST(HardwareAffinity, MempolicyRejectsUnknownStringThrows) { + cudaqx::heterogeneous_map p; + p.insert("mempolicy", std::string("bnid")); + EXPECT_THROW(cudaq::qec::read_mempolicy(p), std::runtime_error); +} +TEST(HardwareAffinity, NumaNode64ThrowsOnMempolicyBind) { + namespace da = cudaq::qec::detail_affinity; + EXPECT_THROW(da::bind_this_thread_to_numa_node(64), std::runtime_error); +} +TEST(HardwareAffinity, CpuAffinityPinsToExactCores) { + namespace da = cudaq::qec::detail_affinity; + cpu_set_t saved; CPU_ZERO(&saved); sched_getaffinity(0, sizeof(saved), &saved); + da::set_thread_cpu_affinity({0, 2}); + auto cpus = da::current_thread_cpuset(); + EXPECT_EQ(cpus, (std::vector{0, 2})); + sched_setaffinity(0, sizeof(saved), &saved); +} +TEST(HardwareAffinity, CpuAffinityEmptyIsNoop) { + namespace da = cudaq::qec::detail_affinity; + cpu_set_t before; CPU_ZERO(&before); sched_getaffinity(0, sizeof(before), &before); + da::set_thread_cpu_affinity({}); + cpu_set_t after; CPU_ZERO(&after); sched_getaffinity(0, sizeof(after), &after); + EXPECT_TRUE(CPU_EQUAL(&before, &after)); +} +TEST(HardwareAffinity, CpuAffinityOutOfRangeCoreThrows) { + namespace da = cudaq::qec::detail_affinity; + EXPECT_THROW(da::set_thread_cpu_affinity({CPU_SETSIZE}), std::invalid_argument); + EXPECT_THROW(da::set_thread_cpu_affinity({-1}), std::invalid_argument); +} +TEST(HardwareAffinity, BindThreadPinsAffinityToNodeCpus) { + namespace da = cudaq::qec::detail_affinity; + cpu_set_t node0; CPU_ZERO(&node0); + if (!da::build_node_cpuset(0, node0)) GTEST_SKIP() << "no node0 cpulist"; + cpu_set_t saved; CPU_ZERO(&saved); sched_getaffinity(0, sizeof(saved), &saved); + da::bind_this_thread_to_numa_node(0); + cpu_set_t have; CPU_ZERO(&have); sched_getaffinity(0, sizeof(have), &have); + for (int c = 0; c < CPU_SETSIZE; ++c) + if (CPU_ISSET(c, &have)) EXPECT_TRUE(CPU_ISSET(c, &node0)) << "cpu " << c << " not on node 0"; + sched_setaffinity(0, sizeof(saved), &saved); + syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); +} +TEST(HardwareAffinity, NegativeNodeIsNoop) { + namespace da = cudaq::qec::detail_affinity; + cpu_set_t before; CPU_ZERO(&before); sched_getaffinity(0, sizeof(before), &before); + da::bind_this_thread_to_numa_node(-1); + cpu_set_t after; CPU_ZERO(&after); sched_getaffinity(0, sizeof(after), &after); + EXPECT_TRUE(CPU_EQUAL(&before, &after)); + EXPECT_EQ(da::current_thread_mempolicy_mode(), MPOL_DEFAULT); +} +TEST(HardwareAffinity, BindRegionSetsPolicyOnBuffer) { + namespace da = cudaq::qec::detail_affinity; + const size_t bytes = 4096; + void *p = std::calloc(1, bytes); + ASSERT_NE(p, nullptr); + da::bind_region_to_numa_node(p, bytes, 0); // preferred, node 0 + int mode = -1; + long rc = syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, p, 1UL /*MPOL_F_ADDR*/); + if (rc == 0) + EXPECT_TRUE(mode == MPOL_PREFERRED || mode == MPOL_DEFAULT) + << "region policy after preferred-bind should be PREFERRED (or DEFAULT if unsupported)"; + da::bind_region_to_numa_node(p, bytes, -1); // negative node -> no-op, no crash + std::free(p); +} +TEST(HardwareAffinity, BindRegionNode64Throws) { + namespace da = cudaq::qec::detail_affinity; + void *p = std::calloc(1, 4096); ASSERT_NE(p, nullptr); + EXPECT_THROW(da::bind_region_to_numa_node(p, 4096, 64), std::runtime_error); + std::free(p); +} +#endif From 6069dcbaad0a29cec22b1863357284cd35cb765b Mon Sep 17 00:00:00 2001 From: kvmto Date: Wed, 1 Jul 2026 15:50:58 +0000 Subject: [PATCH 04/16] feat(qec): concurrent multi-decoder pool with per-GPU pinning Add decoder_pool: runs a set of decoders concurrently, each on its own persistent worker thread pinned (bind_current_thread) to that decoder's CUDA device / NUMA node. Each decoder is constructed on its worker thread so its GPU resources land on-node. decode_all() fans a per-decoder workload out across the workers and aggregates results by id. Also add a pinned-worker decode helper and a loud, cudaq-decoupled warning when a decoder runs on a device that does not match its cuda_device_id. Tests: pool routing/aggregation; construct-time and decode-time GPU placement (eager/lazy probe decoders); two real trt decoders run concurrently across two GPUs; a gated nv-qldpc pool test that skips where the closed decoder is not built against this base. nv-qldpc-placement- derisk.md explains why the installed closed nv-qldpc cannot validate against a modified base. Signed-off-by: kvmto --- libs/qec/include/cudaq/qec/decoder.h | 16 +- libs/qec/include/cudaq/qec/decoder_pool.h | 50 +++++++ libs/qec/include/cudaq/qec/device_affinity.h | 15 +- libs/qec/lib/CMakeLists.txt | 1 + libs/qec/lib/decoder.cpp | 81 ++++++++--- libs/qec/lib/decoder_pool.cpp | 111 ++++++++++++++ libs/qec/lib/hardware_affinity.h | 56 ++++--- libs/qec/unittests/CMakeLists.txt | 6 + .../qec/unittests/decoders/sample_decoder.cpp | 46 +++++- .../decoders/trt_decoder/test_trt_decoder.cpp | 87 +++++++++++ libs/qec/unittests/test_decoder_pool.cpp | 137 ++++++++++++++++++ libs/qec/unittests/test_decoders.cpp | 135 +++++++++++++---- libs/qec/unittests/test_device_affinity.cpp | 54 +++++-- 13 files changed, 701 insertions(+), 94 deletions(-) create mode 100644 libs/qec/include/cudaq/qec/decoder_pool.h create mode 100644 libs/qec/lib/decoder_pool.cpp create mode 100644 libs/qec/unittests/test_decoder_pool.cpp diff --git a/libs/qec/include/cudaq/qec/decoder.h b/libs/qec/include/cudaq/qec/decoder.h index 5be33b7e6..f2ee953c9 100644 --- a/libs/qec/include/cudaq/qec/decoder.h +++ b/libs/qec/include/cudaq/qec/decoder.h @@ -264,6 +264,12 @@ class decoder /// was requested (>= 0) but could not be honored. int bind_current_thread(); + /// @brief Run decode(syndrome) on a fresh worker thread bound (via + /// bind_current_thread) to this decoder's CUDA device / NUMA node, and return + /// the result. Convenience for a one-off pinned decode; for sustained + /// throughput drive decode on a long-lived bound thread instead. + decoder_result decode_on_pinned_thread(const std::vector &syndrome); + // -- Begin realtime decoding API -- // Note: all of the current realtime decoding API is designed to be used with @@ -385,11 +391,19 @@ class decoder bool bound_persistently_ = false; /// NUMA memory-policy mode for this decoder (default soft PREFERRED). cudaq::qec::mempolicy_mode mempolicy_ = cudaq::qec::mempolicy_mode::preferred; - /// Explicit CPU cores for this decoder's owning thread (empty = use node cpuset). + /// Explicit CPU cores for this decoder's owning thread (empty = use node + /// cpuset). std::vector cpu_affinity_; + /// Set once the current-device mismatch warning has fired, so it prints at + /// most once per decoder instance. + bool device_mismatch_warned_ = false; private: decode_result_type result_type_ = decode_result_type::decode_to_errs; + + /// Warn (once) if this decoder has an explicit cuda_device_id and the + /// calling thread's current CUDA device does not match it. + void warn_if_device_mismatch(); }; /// @brief Convert a single soft probability to a hard 0/1 decision. diff --git a/libs/qec/include/cudaq/qec/decoder_pool.h b/libs/qec/include/cudaq/qec/decoder_pool.h new file mode 100644 index 000000000..7cb952fe0 --- /dev/null +++ b/libs/qec/include/cudaq/qec/decoder_pool.h @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ +#pragma once + +#include "cudaq/qec/decoder.h" +#include +#include +#include +#include + +namespace cudaq::qec { + +/// @brief One decoder's placement + construction inputs for a decoder_pool. +struct pool_decoder_spec { + int id; ///< Caller-chosen routing id. + std::string name; ///< Registered decoder name. + cudaqx::tensor H; ///< Parity-check matrix. + cudaqx::heterogeneous_map options; ///< get() options (cuda_device_id, ...). +}; + +/// @brief Runs a set of decoders concurrently, each on its own worker thread +/// bound (bind_current_thread) to that decoder's CUDA device / NUMA node. The +/// decoder is constructed on its worker thread so its resources land on-node. +class decoder_pool { +public: + explicit decoder_pool(std::vector specs); + ~decoder_pool(); + decoder_pool(const decoder_pool &) = delete; + decoder_pool &operator=(const decoder_pool &) = delete; + + /// @brief Decode every id's syndromes on that id's pinned worker, with all + /// decoders running concurrently; blocks until they finish and returns the + /// results grouped by id. Each id maps to that decoder's syndromes (the + /// per-decoder batched decode happens inside its worker). Throws if an id has + /// no matching decoder, or rethrows a worker's decode exception. + std::unordered_map> decode_all( + const std::unordered_map>> &work); + +private: + struct worker; + std::vector> workers_; + std::unordered_map by_id_; +}; + +} // namespace cudaq::qec diff --git a/libs/qec/include/cudaq/qec/device_affinity.h b/libs/qec/include/cudaq/qec/device_affinity.h index 780a4ec32..1c6a012c6 100644 --- a/libs/qec/include/cudaq/qec/device_affinity.h +++ b/libs/qec/include/cudaq/qec/device_affinity.h @@ -43,7 +43,8 @@ inline int read_numa_node_id(const cudaqx::heterogeneous_map ¶ms) { /// @brief Public NUMA memory-policy selector. preferred = soft; bind = strict. enum class mempolicy_mode { preferred, bind }; -/// @brief Read "mempolicy": "bind"->bind, "preferred"/absent->preferred, else throw. +/// @brief Read "mempolicy": "bind"->bind, "preferred"/absent->preferred, else +/// throw. inline mempolicy_mode read_mempolicy(const cudaqx::heterogeneous_map ¶ms) { if (!params.contains("mempolicy")) return mempolicy_mode::preferred; @@ -52,18 +53,22 @@ inline mempolicy_mode read_mempolicy(const cudaqx::heterogeneous_map ¶ms) { return mempolicy_mode::bind; if (v == "preferred") return mempolicy_mode::preferred; - throw std::runtime_error("mempolicy must be \"preferred\" or \"bind\" (got \"" + v + "\")"); + throw std::runtime_error( + "mempolicy must be \"preferred\" or \"bind\" (got \"" + v + "\")"); } -/// @brief Read "cpu_affinity": a list of CPU core ids. Absent -> empty (no override). -inline std::vector read_cpu_affinity(const cudaqx::heterogeneous_map ¶ms) { +/// @brief Read "cpu_affinity": a list of CPU core ids. Absent -> empty (no +/// override). +inline std::vector +read_cpu_affinity(const cudaqx::heterogeneous_map ¶ms) { if (!params.contains("cpu_affinity")) return {}; return params.get>("cpu_affinity"); } /// @brief NUMA node local to a CUDA device (via PCIe locality), or -1 if the -/// device id is negative or the topology can't be resolved. Defined in decoder.cpp. +/// device id is negative or the topology can't be resolved. Defined in +/// decoder.cpp. int numa_node_for_cuda_device(int cuda_device_id); } // namespace cudaq::qec diff --git a/libs/qec/lib/CMakeLists.txt b/libs/qec/lib/CMakeLists.txt index 0c34b8015..3f4e6ebad 100644 --- a/libs/qec/lib/CMakeLists.txt +++ b/libs/qec/lib/CMakeLists.txt @@ -41,6 +41,7 @@ endif() set(QEC_SOURCES code.cpp decoder.cpp + decoder_pool.cpp detector_error_model.cpp experiments.cpp pcm_utils.cpp diff --git a/libs/qec/lib/decoder.cpp b/libs/qec/lib/decoder.cpp index 86432a7a3..fe8be0dae 100644 --- a/libs/qec/lib/decoder.cpp +++ b/libs/qec/lib/decoder.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #if defined(__linux__) #include @@ -54,12 +55,16 @@ struct CudaDeviceGuard { throw std::runtime_error( "cuda_device_id " + std::to_string(target) + " out of range (device_count=" + std::to_string(count) + ")"); - if (cudaGetDevice(&prev_) != cudaSuccess) { prev_ = -1; return; } // can't safely switch + if (cudaGetDevice(&prev_) != cudaSuccess) { + prev_ = -1; + return; + } // can't safely switch if (prev_ != target) { cudaError_t e = cudaSetDevice(target); if (e != cudaSuccess) - throw std::runtime_error("CudaDeviceGuard: cudaSetDevice(" + std::to_string(target) + - ") failed: " + cudaGetErrorString(e)); + throw std::runtime_error("CudaDeviceGuard: cudaSetDevice(" + + std::to_string(target) + + ") failed: " + cudaGetErrorString(e)); active_ = true; } } @@ -81,8 +86,9 @@ struct NumaGuard { bool mempol_set_ = false; cpu_set_t prev_set_{}; - explicit NumaGuard(int node, cudaq::qec::detail_affinity::mempolicy_mode mode = - cudaq::qec::detail_affinity::mempolicy_mode::preferred) { + explicit NumaGuard( + int node, cudaq::qec::detail_affinity::mempolicy_mode mode = + cudaq::qec::detail_affinity::mempolicy_mode::preferred) { if (node < 0) return; CPU_ZERO(&prev_set_); @@ -111,8 +117,9 @@ struct NumaGuard { }; #else struct NumaGuard { - explicit NumaGuard(int node, cudaq::qec::detail_affinity::mempolicy_mode mode = - cudaq::qec::detail_affinity::mempolicy_mode::preferred) { + explicit NumaGuard( + int node, cudaq::qec::detail_affinity::mempolicy_mode mode = + cudaq::qec::detail_affinity::mempolicy_mode::preferred) { (void)mode; if (node < 0) return; @@ -203,7 +210,8 @@ int numa_node_for_cuda_device(int cuda_device_id) { return -1; #if defined(__linux__) char busid[32] = {0}; - if (cudaDeviceGetPCIBusId(busid, sizeof(busid), cuda_device_id) != cudaSuccess) { + if (cudaDeviceGetPCIBusId(busid, sizeof(busid), cuda_device_id) != + cudaSuccess) { cudaq::qec::detail_affinity::affinity_info( "numa_node_id auto-derive for cuda_device_id " + std::to_string(cuda_device_id) + @@ -233,7 +241,8 @@ void decoder::set_hardware_params(const cudaqx::heterogeneous_map ¶ms) { numa_node_id_ = cudaq::qec::read_numa_node_id(params); mempolicy_ = cudaq::qec::read_mempolicy(params); cpu_affinity_ = cudaq::qec::read_cpu_affinity(params); - // Soft-derive: user pinned a GPU but not a node -> use the GPU's node (or -1 if unresolved). + // Soft-derive: user pinned a GPU but not a node -> use the GPU's node (or -1 + // if unresolved). if (numa_node_id_ < 0 && cuda_device_id_ >= 0) numa_node_id_ = numa_node_for_cuda_device(cuda_device_id_); } @@ -244,12 +253,14 @@ int decoder::bind_current_thread() { // set directly here and let it stick). int count = 0; if (cudaGetDeviceCount(&count) != cudaSuccess || cuda_device_id_ >= count) - throw std::runtime_error("cuda_device_id " + std::to_string(cuda_device_id_) + + throw std::runtime_error( + "cuda_device_id " + std::to_string(cuda_device_id_) + " out of range or CUDA unavailable in bind_current_thread"); cudaError_t e = cudaSetDevice(cuda_device_id_); if (e != cudaSuccess) throw std::runtime_error("bind_current_thread: cudaSetDevice(" + - std::to_string(cuda_device_id_) + ") failed: " + cudaGetErrorString(e)); + std::to_string(cuda_device_id_) + + ") failed: " + cudaGetErrorString(e)); } cudaq::qec::detail_affinity::bind_this_thread_to_numa_node( numa_node_id_, to_affinity_mode(mempolicy_)); @@ -282,9 +293,42 @@ int decoder::bind_current_thread() { return numa_node_id_; } +decoder_result +decoder::decode_on_pinned_thread(const std::vector &syndrome) { + decoder_result result; + std::exception_ptr err; + std::thread worker([&] { + try { + bind_current_thread(); + result = decode(syndrome); + } catch (...) { + err = std::current_exception(); + } + }); + worker.join(); + if (err) + std::rethrow_exception(err); // surface an OOB-device throw to the caller + return result; +} + +void decoder::warn_if_device_mismatch() { + if (cuda_device_id_ < 0 || device_mismatch_warned_) + return; + int current = -1; + if (cudaGetDevice(¤t) == cudaSuccess && current != cuda_device_id_) { + cudaq::qec::detail_affinity::affinity_warn( + "decoder pinned to cuda_device_id " + std::to_string(cuda_device_id_) + + " but decoding on current device " + std::to_string(current) + + "; call bind_current_thread() on this thread, or use " + "decode_batch/decode_async"); + device_mismatch_warned_ = true; + } +} + // Provide a trivial implementation of for tensor decode call. Child // classes should override this if they never want to pass through floats. decoder_result decoder::decode(const cudaqx::tensor &syndrome) { + warn_if_device_mismatch(); // Check tensor is of order-1 // If order >1, we could check that other modes are of dim = 1 such that // n x 1, or 1 x n tensors are still valid. @@ -308,7 +352,7 @@ decoder::decode_batch(const std::vector> &syndrome) { // per-call guards are redundant set/restore syscalls — skip them. CudaDeviceGuard dev(bound_persistently_ ? -1 : cuda_device_id_); NumaGuard numa(bound_persistently_ ? -1 : numa_node_id_, - to_affinity_mode(mempolicy_)); + to_affinity_mode(mempolicy_)); std::vector result; result.reserve(syndrome.size()); for (auto &s : syndrome) @@ -330,12 +374,12 @@ decoder::decode_async(const std::vector &syndrome) { const int cuda_id = cuda_device_id_; const int numa_id = numa_node_id_; const cudaq::qec::mempolicy_mode mempolicy = mempolicy_; - return std::async(std::launch::async, [this, syndrome, cuda_id, numa_id, - mempolicy] { - CudaDeviceGuard dev(cuda_id); - NumaGuard numa(numa_id, to_affinity_mode(mempolicy)); - return this->decode(syndrome); - }); + return std::async(std::launch::async, + [this, syndrome, cuda_id, numa_id, mempolicy] { + CudaDeviceGuard dev(cuda_id); + NumaGuard numa(numa_id, to_affinity_mode(mempolicy)); + return this->decode(syndrome); + }); } std::unique_ptr @@ -578,6 +622,7 @@ bool decoder::enqueue_syndrome(const uint8_t *syndrome, // Send the data to the decoder. convert_vec_hard_to_soft(pimpl->persistent_detector_buffer, pimpl->persistent_soft_detector_buffer); + warn_if_device_mismatch(); auto decoded_result = decode(pimpl->persistent_soft_detector_buffer); // If we didn't get a decoded result, just return diff --git a/libs/qec/lib/decoder_pool.cpp b/libs/qec/lib/decoder_pool.cpp new file mode 100644 index 000000000..8821a991d --- /dev/null +++ b/libs/qec/lib/decoder_pool.cpp @@ -0,0 +1,111 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ +#include "cudaq/qec/decoder_pool.h" + +#include +#include +#include +#include +#include +#include + +namespace cudaq::qec { + +// One persistent, GPU/NUMA-pinned worker owning a single decoder and a job +// queue. The decoder is constructed and bound on this worker's own thread. +struct decoder_pool::worker { + struct job { + std::vector> syndromes; // owned copy (crosses threads) + std::promise> result; + }; + + pool_decoder_spec spec; + std::mutex m; + std::condition_variable cv; + bool stop = false; + std::queue> jobs; + std::thread thread; + + explicit worker(pool_decoder_spec s) : spec(std::move(s)) { + thread = std::thread([this] { run(); }); + } + ~worker() { + { + std::lock_guard lk(m); + stop = true; + } + cv.notify_all(); + if (thread.joinable()) + thread.join(); + } + + std::future> + submit(std::vector> syndromes) { + auto j = std::make_unique(); + j->syndromes = std::move(syndromes); + auto fut = j->result.get_future(); + { + std::lock_guard lk(m); + jobs.push(std::move(j)); + } + cv.notify_one(); + return fut; + } + + void run() { + // Construct + pin on this worker thread so resources land on the target + // GPU. + auto dec = decoder::get(spec.name, spec.H, spec.options); + dec->bind_current_thread(); + for (;;) { + std::unique_ptr j; + { + std::unique_lock lk(m); + cv.wait(lk, [this] { return stop || !jobs.empty(); }); + if (stop && jobs.empty()) + return; + j = std::move(jobs.front()); + jobs.pop(); + } + try { + j->result.set_value(dec->decode_batch(j->syndromes)); + } catch (...) { + j->result.set_exception(std::current_exception()); + } + } + } +}; + +decoder_pool::decoder_pool(std::vector specs) { + workers_.reserve(specs.size()); + for (auto &s : specs) { + int id = s.id; + workers_.push_back(std::make_unique(std::move(s))); + by_id_[id] = workers_.back().get(); + } +} + +decoder_pool::~decoder_pool() = default; // worker dtors stop + join + +std::unordered_map> decoder_pool::decode_all( + const std::unordered_map>> &work) { + std::unordered_map>> futures; + for (const auto &[id, syndromes] : work) { + auto it = by_id_.find(id); + if (it == by_id_.end()) + throw std::runtime_error("decoder_pool::decode_all: no decoder with id " + + std::to_string(id)); + futures.emplace(id, it->second->submit(syndromes)); + } + std::unordered_map> results; + for (auto &[id, fut] : futures) + results.emplace(id, fut.get()); // rethrows any worker exception + return results; +} + +} // namespace cudaq::qec diff --git a/libs/qec/lib/hardware_affinity.h b/libs/qec/lib/hardware_affinity.h index 81665a0d3..ecdbc6c83 100644 --- a/libs/qec/lib/hardware_affinity.h +++ b/libs/qec/lib/hardware_affinity.h @@ -31,12 +31,14 @@ namespace cudaq::qec::detail_affinity { -// Decoupled diagnostics: plain C, no cudaq dependency, so this header can be reused independently. +// Decoupled diagnostics: plain C, no cudaq dependency, so this header can be +// reused independently. inline void affinity_warn(const std::string &msg) { std::fprintf(stderr, "[cudaq-qec affinity] WARNING: %s\n", msg.c_str()); } -// INFO/debug: only shown when CUDAQ_QEC_AFFINITY_DEBUG is set (a legitimate no-op like a -// single-node auto-derive must not spam; it is debug-loggable, not a failure). +// INFO/debug: only shown when CUDAQ_QEC_AFFINITY_DEBUG is set (a legitimate +// no-op like a single-node auto-derive must not spam; it is debug-loggable, not +// a failure). inline void affinity_info(const std::string &msg) { if (std::getenv("CUDAQ_QEC_AFFINITY_DEBUG") != nullptr) std::fprintf(stderr, "[cudaq-qec affinity] INFO: %s\n", msg.c_str()); @@ -83,15 +85,16 @@ inline bool build_node_cpuset(int node, cpu_set_t &out) { // affinity is readable (avoids permanent pinning in a locked-cpuset container). // Throws if node cannot be encoded in the mempolicy nodemask (node >= 64). // Warns (does not throw) if the OS declines an otherwise well-formed request. -inline void bind_this_thread_to_numa_node(int node, - mempolicy_mode mode = mempolicy_mode::preferred) { +inline void +bind_this_thread_to_numa_node(int node, + mempolicy_mode mode = mempolicy_mode::preferred) { if (node < 0) return; if (node >= static_cast(sizeof(unsigned long) * 8)) - throw std::runtime_error( - "numa_node_id " + std::to_string(node) + - " exceeds the maximum encodable NUMA node (" + - std::to_string(sizeof(unsigned long) * 8 - 1) + "); cannot bind memory policy"); + throw std::runtime_error("numa_node_id " + std::to_string(node) + + " exceeds the maximum encodable NUMA node (" + + std::to_string(sizeof(unsigned long) * 8 - 1) + + "); cannot bind memory policy"); cpu_set_t prev; CPU_ZERO(&prev); const bool can_restore = (sched_getaffinity(0, sizeof(prev), &prev) == 0); @@ -99,9 +102,10 @@ inline void bind_this_thread_to_numa_node(int node, CPU_ZERO(&node_set); if (can_restore && build_node_cpuset(node, node_set)) { if (sched_setaffinity(0, sizeof(node_set), &node_set) != 0) - affinity_warn("numa_node_id " + std::to_string(node) + - " requested but sched_setaffinity could not pin the thread: " + - std::strerror(errno) + "; running unpinned (locked cpuset?)"); + affinity_warn( + "numa_node_id " + std::to_string(node) + + " requested but sched_setaffinity could not pin the thread: " + + std::strerror(errno) + "; running unpinned (locked cpuset?)"); } else { affinity_warn("numa_node_id " + std::to_string(node) + " requested but its CPU list could not be read (or prior " @@ -111,8 +115,8 @@ inline void bind_this_thread_to_numa_node(int node, if (syscall(SYS_set_mempolicy, mempolicy_syscall_mode(mode), &nodemask, static_cast(sizeof(nodemask) * 8)) != 0) affinity_warn("numa_node_id " + std::to_string(node) + - " requested but set_mempolicy failed: " + std::strerror(errno) + - "; memory not bound to node"); + " requested but set_mempolicy failed: " + + std::strerror(errno) + "; memory not bound to node"); } // Migrate an already-allocated region onto a NUMA node. The ring buffers are @@ -120,8 +124,9 @@ inline void bind_this_thread_to_numa_node(int node, // to relocate the pages. node < 0 / null / 0 bytes = no-op. // Throws if node cannot be encoded in the mempolicy nodemask (node >= 64). // Warns (does not throw) if the OS declines an otherwise well-formed request. -inline void bind_region_to_numa_node(void *p, std::size_t bytes, int node, - mempolicy_mode mode = mempolicy_mode::preferred) { +inline void +bind_region_to_numa_node(void *p, std::size_t bytes, int node, + mempolicy_mode mode = mempolicy_mode::preferred) { if (node < 0 || p == nullptr || bytes == 0) return; if (node >= static_cast(sizeof(unsigned long) * 8)) @@ -131,7 +136,8 @@ inline void bind_region_to_numa_node(void *p, std::size_t bytes, int node, unsigned long nodemask = 1UL << node; if (syscall(SYS_mbind, p, static_cast(bytes), mempolicy_syscall_mode(mode), &nodemask, - static_cast(sizeof(nodemask) * 8), MPOL_MF_MOVE) != 0) + static_cast(sizeof(nodemask) * 8), + MPOL_MF_MOVE) != 0) affinity_warn("mbind of region to numa_node_id " + std::to_string(node) + " failed: " + std::strerror(errno) + "; pages not migrated (missing CAP_SYS_NICE?)"); @@ -156,7 +162,8 @@ inline void set_thread_cpu_affinity(const std::vector &cores) { for (int c : cores) if (c < 0 || c >= CPU_SETSIZE) throw std::invalid_argument("cpu_affinity core id " + std::to_string(c) + - " is out of range [0, " + std::to_string(CPU_SETSIZE) + ")"); + " is out of range [0, " + + std::to_string(CPU_SETSIZE) + ")"); cpu_set_t set; CPU_ZERO(&set); for (int c : cores) @@ -184,18 +191,21 @@ inline std::vector current_thread_cpuset() { #else // non-Linux: no-ops enum class mempolicy_mode { preferred, bind }; -inline void bind_this_thread_to_numa_node(int node, - mempolicy_mode = mempolicy_mode::preferred) { +inline void +bind_this_thread_to_numa_node(int node, + mempolicy_mode = mempolicy_mode::preferred) { if (node < 0) return; static bool warned = false; if (!warned) { - affinity_warn("numa_node_id ignored: NUMA binding is only supported on Linux"); + affinity_warn( + "numa_node_id ignored: NUMA binding is only supported on Linux"); warned = true; } } -inline void bind_region_to_numa_node(void *, std::size_t, int, - mempolicy_mode = mempolicy_mode::preferred) {} +inline void +bind_region_to_numa_node(void *, std::size_t, int, + mempolicy_mode = mempolicy_mode::preferred) {} inline int current_thread_mempolicy_mode() { return -1; } inline void set_thread_cpu_affinity(const std::vector &) {} inline std::vector current_thread_cpuset() { return {}; } diff --git a/libs/qec/unittests/CMakeLists.txt b/libs/qec/unittests/CMakeLists.txt index 173e3e76a..ea02473b3 100644 --- a/libs/qec/unittests/CMakeLists.txt +++ b/libs/qec/unittests/CMakeLists.txt @@ -40,6 +40,12 @@ target_include_directories(test_decoders PRIVATE ${CUDAToolkit_INCLUDE_DIRS}) add_dependencies(CUDAQXQECUnitTests test_decoders) gtest_discover_tests(test_decoders) +add_executable(test_decoder_pool test_decoder_pool.cpp decoders/sample_decoder.cpp) +target_link_libraries(test_decoder_pool PRIVATE GTest::gtest_main cudaq-qec cudaq::cudaq libstim CUDA::cudart) +target_include_directories(test_decoder_pool PRIVATE ${CUDAToolkit_INCLUDE_DIRS}) +add_dependencies(CUDAQXQECUnitTests test_decoder_pool) +gtest_discover_tests(test_decoder_pool) + add_executable(test_decoders_yaml test_decoders_yaml.cpp decoders/sample_decoder.cpp) target_link_libraries(test_decoders_yaml PRIVATE GTest::gtest_main diff --git a/libs/qec/unittests/decoders/sample_decoder.cpp b/libs/qec/unittests/decoders/sample_decoder.cpp index bbc4c52f9..9bc9efae3 100644 --- a/libs/qec/unittests/decoders/sample_decoder.cpp +++ b/libs/qec/unittests/decoders/sample_decoder.cpp @@ -70,7 +70,7 @@ class device_probe_decoder : public decoder { if (cudaMalloc(&p, 16) == cudaSuccess && p) { cudaPointerAttributes attr{}; if (cudaPointerGetAttributes(&attr, p) == cudaSuccess) - dev = attr.device; // device the lazy allocation landed on + dev = attr.device; // device the lazy allocation landed on cudaFree(p); } else { cudaGetDevice(&dev); @@ -91,4 +91,48 @@ class device_probe_decoder : public decoder { CUDAQ_EXT_PT_REGISTER_TYPE(device_probe_decoder) +/// @brief Test-only decoder shaped like a typical GPU decoder that allocates +/// eagerly at construction (like production GPU decoders), so a test can +/// assert the construct-time device guard placed it correctly. The device its +/// constructor-time allocation landed on is recorded in a member and echoed +/// back by decode() via result.result[0]. +class eager_device_probe_decoder : public decoder { +private: + int device_ = -1; + +public: + eager_device_probe_decoder(const cudaq::qec::sparse_binary_matrix &H, + const cudaqx::heterogeneous_map ¶ms) + : decoder(H) { + void *p = nullptr; + if (cudaMalloc(&p, 16) == cudaSuccess && p) { + cudaPointerAttributes attr{}; + if (cudaPointerGetAttributes(&attr, p) == cudaSuccess) + device_ = attr.device; // device the eager allocation landed on + cudaFree(p); + } else { + cudaGetDevice(&device_); + } + } + + virtual decoder_result decode(const std::vector &syndrome) override { + decoder_result result; + result.converged = true; + result.result = std::vector{static_cast(device_)}; + return result; + } + + virtual ~eager_device_probe_decoder() {} + + CUDAQ_EXTENSION_CUSTOM_CREATOR_FUNCTION( + eager_device_probe_decoder, static std::unique_ptr create( + const cudaq::qec::decoder_init &init, + const cudaqx::heterogeneous_map ¶ms) { + return cudaq::qec::make_pcm_decoder(init, + params); + }) +}; + +CUDAQ_EXT_PT_REGISTER_TYPE(eager_device_probe_decoder) + } // namespace cudaq::qec diff --git a/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp b/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp index 1a9f24113..ad7a00da5 100644 --- a/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp +++ b/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp @@ -8,6 +8,7 @@ #include "trt_test_data.h" #include "cudaq/qec/decoder.h" +#include "cudaq/qec/decoder_pool.h" #include "cudaq/qec/trt_decoder_internal.h" #include #include @@ -798,3 +799,89 @@ TEST_F(TRTDecoderTest, CudaDeviceId_DecodeAsyncOnGpu1) { EXPECT_LT(error, 1e-4f) << "GPU-1 decode differs from expected: got " << trt_output << ", expected " << expected_output; } + +// Composition check: two real trt decoders, each configured for a separate GPU, +// run concurrently through the pool and both converge to the correct output. +// This exercises a heavy production decoder end-to-end (not a probe). The +// precise per-worker device placement is asserted by the eager-probe pool test; +// here the value is that a real decoder composes with the pool and stays +// correct. +TEST_F(TRTDecoderTest, PoolRunsTwoTrtDecodersConcurrently) { + if (!gpu_available()) + GTEST_SKIP() << "No CUDA GPU available"; + int count = 0; + cudaGetDeviceCount(&count); + if (count < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; + std::string onnx_path = get_onnx_asset_path(); + if (!std::filesystem::exists(onnx_path)) + GTEST_SKIP() << "ONNX model not found: " << onnx_path; + + cudaSetDevice(0); // calling thread stays on the default device + + std::size_t num_detectors = NUM_DETECTORS; + cudaqx::tensor H_mat({num_detectors, num_detectors}); + for (std::size_t i = 0; i < num_detectors; ++i) + H_mat.at({i, i}) = 1; + + cudaqx::heterogeneous_map params0; + params0.insert("onnx_load_path", onnx_path); + params0.insert("cuda_device_id", 0); + cudaqx::heterogeneous_map params1; + params1.insert("onnx_load_path", onnx_path); + params1.insert("cuda_device_id", 1); + + std::vector specs; + specs.push_back({0, "trt_decoder", H_mat, params0}); + specs.push_back({1, "trt_decoder", H_mat, params1}); + + // Informational only: engine memory landing on each assigned GPU is + // reported for a human to see, but deltas are noisy so we don't assert. + auto free_bytes = [](int device) -> std::size_t { + int prev = 0; + cudaGetDevice(&prev); + cudaSetDevice(device); + std::size_t free = 0, total = 0; + cudaMemGetInfo(&free, &total); + cudaSetDevice(prev); + return free; + }; + std::size_t free0_before = free_bytes(0); + std::size_t free1_before = free_bytes(1); + + std::unique_ptr pool; + try { + pool = std::make_unique(std::move(specs)); + } catch (const std::exception &e) { + GTEST_SKIP() << "TRT construction failed: " << e.what(); + } + + std::size_t free0_after = free_bytes(0); + std::size_t free1_after = free_bytes(1); + std::cout << "GPU 0 free-memory drop: " + << (free0_before - free0_after) / (1024 * 1024) << " MiB\n"; + std::cout << "GPU 1 free-memory drop: " + << (free1_before - free1_after) / (1024 * 1024) << " MiB\n"; + + std::vector syndrome(TEST_INPUTS[0].begin(), + TEST_INPUTS[0].end()); + std::unordered_map>> work; + work[0] = {syndrome}; + work[1] = {syndrome}; + auto res = pool->decode_all(work); + + ASSERT_FALSE(res[0].empty()); + ASSERT_FALSE(res[1].empty()); + EXPECT_TRUE(res[0][0].converged); + EXPECT_TRUE(res[1][0].converged); + ASSERT_FALSE(res[0][0].result.empty()); + ASSERT_FALSE(res[1][0].result.empty()); + + float expected_output = TEST_OUTPUTS[0][0]; + EXPECT_LT(std::abs(res[0][0].result[0] - expected_output), 1e-4f) + << "GPU-0 pool decode differs from expected: got " << res[0][0].result[0] + << ", expected " << expected_output; + EXPECT_LT(std::abs(res[1][0].result[0] - expected_output), 1e-4f) + << "GPU-1 pool decode differs from expected: got " << res[1][0].result[0] + << ", expected " << expected_output; +} diff --git a/libs/qec/unittests/test_decoder_pool.cpp b/libs/qec/unittests/test_decoder_pool.cpp new file mode 100644 index 000000000..79c72de98 --- /dev/null +++ b/libs/qec/unittests/test_decoder_pool.cpp @@ -0,0 +1,137 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ +#include "cudaq/qec/decoder_pool.h" +#include +#include + +static cudaqx::tensor makeH() { + cudaqx::tensor H({2, 3}); + return H; +} + +// Orchestration (no GPU needed): each id's syndromes are routed to its own +// decoder and aggregated back under that id. +TEST(DecoderPool, RoutesAndAggregatesPerId) { + using cudaq::qec::float_t; + std::vector specs; + specs.push_back({7, "multi_error_lut", makeH(), {}}); + specs.push_back({9, "multi_error_lut", makeH(), {}}); + cudaq::qec::decoder_pool pool(std::move(specs)); + + std::unordered_map>> work; + work[7] = {{0.1f, 0.1f}, {0.2f, 0.2f}}; + work[9] = {{0.3f, 0.3f}}; + auto res = pool.decode_all(work); + ASSERT_EQ(res.size(), 2u); + EXPECT_EQ(res[7].size(), 2u); + EXPECT_EQ(res[9].size(), 1u); +} + +TEST(DecoderPool, UnknownIdThrows) { + using cudaq::qec::float_t; + std::vector specs; + specs.push_back({1, "multi_error_lut", makeH(), {}}); + cudaq::qec::decoder_pool pool(std::move(specs)); + std::unordered_map>> work; + work[42] = {{0.1f, 0.1f}}; + EXPECT_THROW(pool.decode_all(work), std::runtime_error); +} + +// Placement: each worker decodes on its assigned GPU. Uses the probe decoder, +// which reports (via result.result[0]) the device its allocation landed on. +TEST(DecoderPool, EachWorkerRunsOnAssignedGpu) { + using cudaq::qec::float_t; + int n = 0; + cudaGetDeviceCount(&n); + if (n < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); + std::vector specs; + cudaqx::heterogeneous_map o0; + o0.insert("cuda_device_id", 0); + cudaqx::heterogeneous_map o1; + o1.insert("cuda_device_id", 1); + specs.push_back({0, "device_probe_decoder", makeH(), o0}); + specs.push_back({1, "device_probe_decoder", makeH(), o1}); + cudaq::qec::decoder_pool pool(std::move(specs)); + std::unordered_map>> work; + work[0] = {{0.1f, 0.1f}}; + work[1] = {{0.1f, 0.1f}}; + auto res = pool.decode_all(work); + ASSERT_EQ(res[0].size(), 1u); + ASSERT_EQ(res[1].size(), 1u); + EXPECT_EQ(static_cast(res[0][0].result[0]), 0); + EXPECT_EQ(static_cast(res[1][0].result[0]), 1); +} + +// Validates the real closed nv-qldpc decoder, when compiled against this +// base, running two decoders concurrently on distinct GPUs through the pool. +// Skips wherever nv-qldpc-decoder isn't installed (e.g. this dev build). +TEST(DecoderPool, NvQldpcRunsOnDistinctGpusWhenAvailable) { + using cudaq::qec::float_t; + std::vector H_vec = {1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, + 1, 0, 1, 0, 0, 1, 0, 1, 1, 1}; + cudaqx::tensor H; + H.copy(H_vec.data(), {3, 7}); + try { + auto d = cudaq::qec::decoder::get("nv-qldpc-decoder", H); + } catch (const std::exception &) { + GTEST_SKIP() << "nv-qldpc-decoder not available"; + } + + int n = 0; + cudaGetDeviceCount(&n); + if (n < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; + + std::vector specs; + cudaqx::heterogeneous_map o0; + o0.insert("cuda_device_id", 0); + cudaqx::heterogeneous_map o1; + o1.insert("cuda_device_id", 1); + specs.push_back({0, "nv-qldpc-decoder", H, o0}); + specs.push_back({1, "nv-qldpc-decoder", H, o1}); + cudaq::qec::decoder_pool pool(std::move(specs)); + + std::unordered_map>> work; + work[0] = {{1.0f, 0.0f, 1.0f}}; + work[1] = {{1.0f, 0.0f, 1.0f}}; + auto res = pool.decode_all(work); + ASSERT_EQ(res[0].size(), 1u); + ASSERT_EQ(res[1].size(), 1u); + EXPECT_TRUE(res[0][0].converged); + EXPECT_TRUE(res[1][0].converged); +} + +// Placement at construction time: each worker's decoder allocates GPU memory +// in its constructor (like production GPU decoders), and the construct-time +// device guard must already have it on the right device before any decode. +TEST(DecoderPool, EachWorkerConstructsOnAssignedGpu) { + using cudaq::qec::float_t; + int n = 0; + cudaGetDeviceCount(&n); + if (n < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); + std::vector specs; + cudaqx::heterogeneous_map o0; + o0.insert("cuda_device_id", 0); + cudaqx::heterogeneous_map o1; + o1.insert("cuda_device_id", 1); + specs.push_back({0, "eager_device_probe_decoder", makeH(), o0}); + specs.push_back({1, "eager_device_probe_decoder", makeH(), o1}); + cudaq::qec::decoder_pool pool(std::move(specs)); + std::unordered_map>> work; + work[0] = {{0.1f, 0.1f}}; + work[1] = {{0.1f, 0.1f}}; + auto res = pool.decode_all(work); + ASSERT_EQ(res[0].size(), 1u); + ASSERT_EQ(res[1].size(), 1u); + EXPECT_EQ(static_cast(res[0][0].result[0]), 0); + EXPECT_EQ(static_cast(res[1][0].result[0]), 1); +} diff --git a/libs/qec/unittests/test_decoders.cpp b/libs/qec/unittests/test_decoders.cpp index 1404e0257..d3936b352 100644 --- a/libs/qec/unittests/test_decoders.cpp +++ b/libs/qec/unittests/test_decoders.cpp @@ -1064,7 +1064,9 @@ TEST(HardwarePinning, BindCurrentThreadAppliesCpuAffinity) { std::thread t([&] { d->bind_current_thread(); #if defined(__linux__) - cpu_set_t have; CPU_ZERO(&have); sched_getaffinity(0, sizeof(have), &have); + cpu_set_t have; + CPU_ZERO(&have); + sched_getaffinity(0, sizeof(have), &have); EXPECT_TRUE(CPU_ISSET(0, &have)); EXPECT_EQ(CPU_COUNT(&have), 1); #endif @@ -1082,7 +1084,8 @@ TEST(HardwarePinning, BindCurrentThreadAppliesCpuAffinity) { // fires during get() itself. Assert the throw at the layer where it actually // fires: construction. TEST(HardwarePinning, BindCurrentThreadThrowsOnOutOfRangeDevice) { - int n = 0; cudaGetDeviceCount(&n); + int n = 0; + cudaGetDeviceCount(&n); cudaqx::tensor H({2, 3}); cudaqx::heterogeneous_map params; params.insert("cuda_device_id", n + 100); // guaranteed OOB @@ -1259,15 +1262,18 @@ TEST(SlidingWindowDecoder, BaseStreamingCopiesFirstRoundDetectors) { } // GPU placement: cuda_device_id must be honored across the decode entry points. -// A decoder that overrides only decode() and allocates lazily still lands on the -// assigned GPU when its owning thread is pinned via bind_current_thread(). +// A decoder that overrides only decode() and allocates lazily still lands on +// the assigned GPU when its owning thread is pinned via bind_current_thread(). TEST(HardwarePinningGpu, RawDecodeOnPinnedThreadUsesAssignedGpu) { - int n = 0; cudaGetDeviceCount(&n); - if (n < 2) GTEST_SKIP() << "needs >= 2 GPUs"; + int n = 0; + cudaGetDeviceCount(&n); + if (n < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; cudaSetDevice(0); cudaqx::tensor H({2, 3}); - cudaqx::heterogeneous_map p; p.insert("cuda_device_id", 1); + cudaqx::heterogeneous_map p; + p.insert("cuda_device_id", 1); auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); int observed = -1; std::size_t sz = 0; @@ -1280,18 +1286,23 @@ TEST(HardwarePinningGpu, RawDecodeOnPinnedThreadUsesAssignedGpu) { }); worker.join(); ASSERT_EQ(sz, 1u); - EXPECT_EQ(observed, 1) << "decode() on a pinned worker did not use the assigned GPU"; + EXPECT_EQ(observed, 1) + << "decode() on a pinned worker did not use the assigned GPU"; } -// Two logical patches on two GPUs: each decoder, pinned to its own GPU on its own -// worker thread, decodes on its assigned GPU with no cross-talk. +// Two logical patches on two GPUs: each decoder, pinned to its own GPU on its +// own worker thread, decodes on its assigned GPU with no cross-talk. TEST(HardwarePinningGpu, MultiPatchOnPinnedThreadsUseDistinctGpus) { - int n = 0; cudaGetDeviceCount(&n); - if (n < 2) GTEST_SKIP() << "needs >= 2 GPUs"; + int n = 0; + cudaGetDeviceCount(&n); + if (n < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; cudaSetDevice(0); cudaqx::tensor H({2, 3}); - cudaqx::heterogeneous_map p0; p0.insert("cuda_device_id", 0); - cudaqx::heterogeneous_map p1; p1.insert("cuda_device_id", 1); + cudaqx::heterogeneous_map p0; + p0.insert("cuda_device_id", 0); + cudaqx::heterogeneous_map p1; + p1.insert("cuda_device_id", 1); auto d0 = cudaq::qec::decoder::get("device_probe_decoder", H, p0); auto d1 = cudaq::qec::decoder::get("device_probe_decoder", H, p1); int o0 = -1, o1 = -1; @@ -1313,13 +1324,17 @@ TEST(HardwarePinningGpu, MultiPatchOnPinnedThreadsUseDistinctGpus) { EXPECT_EQ(o1, 1) << "patch 1 did not use GPU 1"; } -// decode_async applies the device guard on its worker, so it honors cuda_device_id. +// decode_async applies the device guard on its worker, so it honors +// cuda_device_id. TEST(HardwarePinningGpu, DecodeAsyncHonorsCudaDeviceId) { - int n = 0; cudaGetDeviceCount(&n); - if (n < 2) GTEST_SKIP() << "needs >= 2 GPUs"; + int n = 0; + cudaGetDeviceCount(&n); + if (n < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; cudaSetDevice(0); cudaqx::tensor H({2, 3}); - cudaqx::heterogeneous_map p; p.insert("cuda_device_id", 1); + cudaqx::heterogeneous_map p; + p.insert("cuda_device_id", 1); auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); auto fut = d->decode_async(std::vector{0.1f, 0.1f}); auto r = fut.get(); @@ -1329,11 +1344,14 @@ TEST(HardwarePinningGpu, DecodeAsyncHonorsCudaDeviceId) { // decode_batch applies the device guard, so it honors cuda_device_id. TEST(HardwarePinningGpu, DecodeBatchHonorsCudaDeviceId) { - int n = 0; cudaGetDeviceCount(&n); - if (n < 2) GTEST_SKIP() << "needs >= 2 GPUs"; + int n = 0; + cudaGetDeviceCount(&n); + if (n < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; cudaSetDevice(0); cudaqx::tensor H({2, 3}); - cudaqx::heterogeneous_map p; p.insert("cuda_device_id", 1); + cudaqx::heterogeneous_map p; + p.insert("cuda_device_id", 1); auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); auto rs = d->decode_batch( std::vector>{{0.1f, 0.1f}}); @@ -1342,22 +1360,79 @@ TEST(HardwarePinningGpu, DecodeBatchHonorsCudaDeviceId) { << "decode_batch did not honor cuda_device_id"; } +// The base-controlled decode(tensor) overload and enqueue_syndrome() both +// check for a mismatch between an explicit cuda_device_id and the caller's +// current CUDA device, warning once on stderr rather than silently decoding +// on the wrong GPU. +TEST(HardwarePinningGpu, WarnsOnceWhenDecodingOnMismatchedDevice) { + int n = 0; + cudaGetDeviceCount(&n); + if (n < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); // current device 0, but decoder wants 1 -> mismatch + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map p; + p.insert("cuda_device_id", 1); + auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); + cudaqx::tensor syn; + std::vector sv = {1, 0}; + syn.copy(sv.data(), {2}); + testing::internal::CaptureStderr(); + (void)d->decode(syn); // base tensor overload -> warn_if_device_mismatch + (void)d->decode(syn); // second call must NOT warn again (once) + std::string err = testing::internal::GetCapturedStderr(); + EXPECT_NE(err.find("cuda_device_id 1"), std::string::npos) + << "expected a device-mismatch warning on stderr"; + // warn-once: the phrase appears exactly once + auto first = err.find("but decoding on current device"); + ASSERT_NE(first, std::string::npos); + EXPECT_EQ(err.find("but decoding on current device", first + 1), + std::string::npos) + << "device-mismatch warning should fire at most once"; +} + +// decode_on_pinned_thread spawns a worker, binds it to the decoder's assigned +// GPU/NUMA node, decodes there, and joins before returning the result. +TEST(HardwarePinningGpu, DecodeOnPinnedThreadUsesAssignedGpu) { + int n = 0; + cudaGetDeviceCount(&n); + if (n < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map p; + p.insert("cuda_device_id", 1); + auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); + auto r = + d->decode_on_pinned_thread(std::vector{0.1f, 0.1f}); + ASSERT_EQ(r.result.size(), 1u); + EXPECT_EQ(static_cast(r.result[0]), 1) + << "decode_on_pinned_thread did not run on the assigned GPU"; +} + TEST(HardwarePinning, NumaDerivesFromCudaDeviceOrDegrades) { - int n = 0; cudaGetDeviceCount(&n); - if (n < 1) GTEST_SKIP() << "needs a GPU"; + int n = 0; + cudaGetDeviceCount(&n); + if (n < 1) + GTEST_SKIP() << "needs a GPU"; int node = cudaq::qec::numa_node_for_cuda_device(0); - EXPECT_GE(node, -1); // real node on multi-node HW, or -1 - EXPECT_EQ(cudaq::qec::numa_node_for_cuda_device(-1), -1); // negative device -> -1 + EXPECT_GE(node, -1); // real node on multi-node HW, or -1 + EXPECT_EQ(cudaq::qec::numa_node_for_cuda_device(-1), + -1); // negative device -> -1 } TEST(HardwarePinning, GetDecoderSoftDerivesNumaWhenOnlyCudaSet) { - int n = 0; cudaGetDeviceCount(&n); - if (n < 1) GTEST_SKIP() << "needs a GPU"; + int n = 0; + cudaGetDeviceCount(&n); + if (n < 1) + GTEST_SKIP() << "needs a GPU"; cudaqx::tensor H({2, 3}); cudaqx::heterogeneous_map params; - params.insert("cuda_device_id", 0); // numa_node_id NOT set + params.insert("cuda_device_id", 0); // numa_node_id NOT set auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); - EXPECT_EQ(d->numa_node_id(), cudaq::qec::numa_node_for_cuda_device(0)); // derivation ran + EXPECT_EQ(d->numa_node_id(), + cudaq::qec::numa_node_for_cuda_device(0)); // derivation ran } TEST(HardwarePinning, NumaAutoDeriveUnknownIsInfoNotThrow) { - EXPECT_NO_THROW({ EXPECT_EQ(cudaq::qec::numa_node_for_cuda_device(-1), -1); }); + EXPECT_NO_THROW( + { EXPECT_EQ(cudaq::qec::numa_node_for_cuda_device(-1), -1); }); } diff --git a/libs/qec/unittests/test_device_affinity.cpp b/libs/qec/unittests/test_device_affinity.cpp index 27a184d55..a899d6a11 100644 --- a/libs/qec/unittests/test_device_affinity.cpp +++ b/libs/qec/unittests/test_device_affinity.cpp @@ -74,7 +74,9 @@ TEST(HardwareAffinity, NumaNode64ThrowsOnMempolicyBind) { } TEST(HardwareAffinity, CpuAffinityPinsToExactCores) { namespace da = cudaq::qec::detail_affinity; - cpu_set_t saved; CPU_ZERO(&saved); sched_getaffinity(0, sizeof(saved), &saved); + cpu_set_t saved; + CPU_ZERO(&saved); + sched_getaffinity(0, sizeof(saved), &saved); da::set_thread_cpu_affinity({0, 2}); auto cpus = da::current_thread_cpuset(); EXPECT_EQ(cpus, (std::vector{0, 2})); @@ -82,33 +84,49 @@ TEST(HardwareAffinity, CpuAffinityPinsToExactCores) { } TEST(HardwareAffinity, CpuAffinityEmptyIsNoop) { namespace da = cudaq::qec::detail_affinity; - cpu_set_t before; CPU_ZERO(&before); sched_getaffinity(0, sizeof(before), &before); + cpu_set_t before; + CPU_ZERO(&before); + sched_getaffinity(0, sizeof(before), &before); da::set_thread_cpu_affinity({}); - cpu_set_t after; CPU_ZERO(&after); sched_getaffinity(0, sizeof(after), &after); + cpu_set_t after; + CPU_ZERO(&after); + sched_getaffinity(0, sizeof(after), &after); EXPECT_TRUE(CPU_EQUAL(&before, &after)); } TEST(HardwareAffinity, CpuAffinityOutOfRangeCoreThrows) { namespace da = cudaq::qec::detail_affinity; - EXPECT_THROW(da::set_thread_cpu_affinity({CPU_SETSIZE}), std::invalid_argument); + EXPECT_THROW(da::set_thread_cpu_affinity({CPU_SETSIZE}), + std::invalid_argument); EXPECT_THROW(da::set_thread_cpu_affinity({-1}), std::invalid_argument); } TEST(HardwareAffinity, BindThreadPinsAffinityToNodeCpus) { namespace da = cudaq::qec::detail_affinity; - cpu_set_t node0; CPU_ZERO(&node0); - if (!da::build_node_cpuset(0, node0)) GTEST_SKIP() << "no node0 cpulist"; - cpu_set_t saved; CPU_ZERO(&saved); sched_getaffinity(0, sizeof(saved), &saved); + cpu_set_t node0; + CPU_ZERO(&node0); + if (!da::build_node_cpuset(0, node0)) + GTEST_SKIP() << "no node0 cpulist"; + cpu_set_t saved; + CPU_ZERO(&saved); + sched_getaffinity(0, sizeof(saved), &saved); da::bind_this_thread_to_numa_node(0); - cpu_set_t have; CPU_ZERO(&have); sched_getaffinity(0, sizeof(have), &have); + cpu_set_t have; + CPU_ZERO(&have); + sched_getaffinity(0, sizeof(have), &have); for (int c = 0; c < CPU_SETSIZE; ++c) - if (CPU_ISSET(c, &have)) EXPECT_TRUE(CPU_ISSET(c, &node0)) << "cpu " << c << " not on node 0"; + if (CPU_ISSET(c, &have)) + EXPECT_TRUE(CPU_ISSET(c, &node0)) << "cpu " << c << " not on node 0"; sched_setaffinity(0, sizeof(saved), &saved); syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); } TEST(HardwareAffinity, NegativeNodeIsNoop) { namespace da = cudaq::qec::detail_affinity; - cpu_set_t before; CPU_ZERO(&before); sched_getaffinity(0, sizeof(before), &before); + cpu_set_t before; + CPU_ZERO(&before); + sched_getaffinity(0, sizeof(before), &before); da::bind_this_thread_to_numa_node(-1); - cpu_set_t after; CPU_ZERO(&after); sched_getaffinity(0, sizeof(after), &after); + cpu_set_t after; + CPU_ZERO(&after); + sched_getaffinity(0, sizeof(after), &after); EXPECT_TRUE(CPU_EQUAL(&before, &after)); EXPECT_EQ(da::current_thread_mempolicy_mode(), MPOL_DEFAULT); } @@ -117,18 +135,22 @@ TEST(HardwareAffinity, BindRegionSetsPolicyOnBuffer) { const size_t bytes = 4096; void *p = std::calloc(1, bytes); ASSERT_NE(p, nullptr); - da::bind_region_to_numa_node(p, bytes, 0); // preferred, node 0 + da::bind_region_to_numa_node(p, bytes, 0); // preferred, node 0 int mode = -1; - long rc = syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, p, 1UL /*MPOL_F_ADDR*/); + long rc = + syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, p, 1UL /*MPOL_F_ADDR*/); if (rc == 0) EXPECT_TRUE(mode == MPOL_PREFERRED || mode == MPOL_DEFAULT) - << "region policy after preferred-bind should be PREFERRED (or DEFAULT if unsupported)"; - da::bind_region_to_numa_node(p, bytes, -1); // negative node -> no-op, no crash + << "region policy after preferred-bind should be PREFERRED (or DEFAULT " + "if unsupported)"; + da::bind_region_to_numa_node(p, bytes, + -1); // negative node -> no-op, no crash std::free(p); } TEST(HardwareAffinity, BindRegionNode64Throws) { namespace da = cudaq::qec::detail_affinity; - void *p = std::calloc(1, 4096); ASSERT_NE(p, nullptr); + void *p = std::calloc(1, 4096); + ASSERT_NE(p, nullptr); EXPECT_THROW(da::bind_region_to_numa_node(p, 4096, 64), std::runtime_error); std::free(p); } From d9917240a5af5196b8cc989d2706980b263dac23 Mon Sep 17 00:00:00 2001 From: kvmto Date: Wed, 1 Jul 2026 16:33:34 +0000 Subject: [PATCH 05/16] feat(qec): non-blocking streaming submit for decoder_pool Add decoder_pool::submit(id, chunk) -> future>: a non-blocking primitive that enqueues a chunk on that id's pinned worker and returns immediately, so callers can push syndromes over time and consume results as they resolve (a size-1 chunk streams a single syndrome). decode_all now routes through submit (all ids submitted before any get, so worker concurrency is unchanged). Tests: interleaved in-flight submits across two decoders (proves submit does not block); streamed chunks to two GPU-pinned workers stay on their assigned device across every chunk. Signed-off-by: kvmto --- libs/qec/include/cudaq/qec/decoder_pool.h | 10 ++++ libs/qec/lib/decoder_pool.cpp | 18 ++++--- libs/qec/unittests/test_decoder_pool.cpp | 59 +++++++++++++++++++++++ 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/libs/qec/include/cudaq/qec/decoder_pool.h b/libs/qec/include/cudaq/qec/decoder_pool.h index 7cb952fe0..6308d1df3 100644 --- a/libs/qec/include/cudaq/qec/decoder_pool.h +++ b/libs/qec/include/cudaq/qec/decoder_pool.h @@ -8,6 +8,7 @@ #pragma once #include "cudaq/qec/decoder.h" +#include #include #include #include @@ -41,6 +42,15 @@ class decoder_pool { std::unordered_map> decode_all( const std::unordered_map>> &work); + /// @brief Non-blocking streaming submit: enqueue `syndromes` on `id`'s pinned + /// worker and return immediately with a future for that chunk's results. + /// Submit chunks over time and consume the futures as they resolve to stream + /// per id, concurrently; a size-1 chunk streams a single syndrome. Throws if + /// `id` has no matching decoder; the future rethrows a worker decode + /// exception. + std::future> + submit(int id, std::vector> syndromes); + private: struct worker; std::vector> workers_; diff --git a/libs/qec/lib/decoder_pool.cpp b/libs/qec/lib/decoder_pool.cpp index 8821a991d..e3260282a 100644 --- a/libs/qec/lib/decoder_pool.cpp +++ b/libs/qec/lib/decoder_pool.cpp @@ -92,16 +92,20 @@ decoder_pool::decoder_pool(std::vector specs) { decoder_pool::~decoder_pool() = default; // worker dtors stop + join +std::future> +decoder_pool::submit(int id, std::vector> syndromes) { + auto it = by_id_.find(id); + if (it == by_id_.end()) + throw std::runtime_error("decoder_pool::submit: no decoder with id " + + std::to_string(id)); + return it->second->submit(std::move(syndromes)); +} + std::unordered_map> decoder_pool::decode_all( const std::unordered_map>> &work) { std::unordered_map>> futures; - for (const auto &[id, syndromes] : work) { - auto it = by_id_.find(id); - if (it == by_id_.end()) - throw std::runtime_error("decoder_pool::decode_all: no decoder with id " + - std::to_string(id)); - futures.emplace(id, it->second->submit(syndromes)); - } + for (const auto &[id, syndromes] : work) + futures.emplace(id, submit(id, syndromes)); // copies chunk into the worker std::unordered_map> results; for (auto &[id, fut] : futures) results.emplace(id, fut.get()); // rethrows any worker exception diff --git a/libs/qec/unittests/test_decoder_pool.cpp b/libs/qec/unittests/test_decoder_pool.cpp index 79c72de98..da32ce107 100644 --- a/libs/qec/unittests/test_decoder_pool.cpp +++ b/libs/qec/unittests/test_decoder_pool.cpp @@ -108,6 +108,65 @@ TEST(DecoderPool, NvQldpcRunsOnDistinctGpusWhenAvailable) { EXPECT_TRUE(res[1][0].converged); } +// Streaming: submit returns immediately without waiting on any decode, so a +// caller can keep several chunks in flight per id at once and drain the +// futures as they resolve. +TEST(DecoderPool, StreamsChunksPerIdConcurrently) { + using cudaq::qec::float_t; + std::vector specs; + specs.push_back({7, "multi_error_lut", makeH(), {}}); + specs.push_back({9, "multi_error_lut", makeH(), {}}); + cudaq::qec::decoder_pool pool(std::move(specs)); + + std::vector< + std::pair>>> + in_flight; + for (int round = 0; round < 4; ++round) { + in_flight.emplace_back(7, pool.submit(7, {{0.1f, 0.1f}, {0.2f, 0.2f}})); + in_flight.emplace_back(9, pool.submit(9, {{0.3f, 0.3f}})); + } + + std::unordered_map counts; + for (auto &[id, fut] : in_flight) + counts[id] += fut.get().size(); + EXPECT_EQ(counts[7], 8u); + EXPECT_EQ(counts[9], 4u); +} + +// Placement across streamed chunks: the worker binds its GPU once at +// construction, so every chunk it decodes over time (not just the first) +// must land on that same device. +TEST(DecoderPool, StreamingKeepsWorkerPinnedAcrossChunks) { + using cudaq::qec::float_t; + int n = 0; + cudaGetDeviceCount(&n); + if (n < 2) + GTEST_SKIP() << "needs >= 2 GPUs"; + cudaSetDevice(0); + std::vector specs; + cudaqx::heterogeneous_map o0; + o0.insert("cuda_device_id", 0); + cudaqx::heterogeneous_map o1; + o1.insert("cuda_device_id", 1); + specs.push_back({0, "device_probe_decoder", makeH(), o0}); + specs.push_back({1, "device_probe_decoder", makeH(), o1}); + cudaq::qec::decoder_pool pool(std::move(specs)); + + std::vector< + std::pair>>> + in_flight; + for (int round = 0; round < 4; ++round) { + in_flight.emplace_back(0, pool.submit(0, {{0.1f, 0.1f}})); + in_flight.emplace_back(1, pool.submit(1, {{0.1f, 0.1f}})); + } + + for (auto &[id, fut] : in_flight) { + auto r = fut.get(); + ASSERT_EQ(r.size(), 1u); + EXPECT_EQ(static_cast(r[0].result[0]), id); + } +} + // Placement at construction time: each worker's decoder allocates GPU memory // in its constructor (like production GPU decoders), and the construct-time // device guard must already have it on the right device before any decode. From 7c2bdac162d63a8810608104383a18592c8eab89 Mon Sep 17 00:00:00 2001 From: kvmto Date: Thu, 2 Jul 2026 11:43:52 +0000 Subject: [PATCH 06/16] fix(qec): decouple plugin params from affinity knobs; add pinning syscall gate - decoder::get(): strip the base-owned affinity keys (cuda_device_id, numa_node_id, mempolicy, cpu_affinity) from the options passed to the plugin constructor. Decoders that strictly validate their parameter keys (e.g. nv-qldpc-decoder) previously rejected them; the base consumes these keys, so plugins never need to see them. Verified with an unmodified nv-qldpc-decoder build: two instances construct and decode concurrently on distinct GPUs through decoder_pool. - realtime: fix the hardware_affinity.h include to be relative to the realtime/ subdirectory; a clean build could not resolve it. - unittests: give test_decoders_yaml the CUDA runtime include/link it needs since sample_decoder.cpp gained the GPU probe decoders. - nv-qldpc pool test: enable use_sparsity (required by its GPU batched path). - add a pinning benchmark lane: an LD_PRELOAD sched_setaffinity counter and a gate asserting a bound decode loop issues zero per-call affinity syscalls (with an unbound canary proving the counter fires), plus a loose A/B throughput report (bound path ~100x faster on a syscall-dominated micro-decode; gate is the deterministic check, timing is report-only). Signed-off-by: kvmto --- libs/qec/lib/decoder.cpp | 13 ++- .../qec/lib/realtime/qec_realtime_session.cpp | 3 +- libs/qec/unittests/CMakeLists.txt | 17 +++- .../support/affinity_syscall_shim.cpp | 31 +++++++ libs/qec/unittests/test_decoder_pool.cpp | 3 + libs/qec/unittests/test_pinning_benchmark.cpp | 84 +++++++++++++++++++ 6 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 libs/qec/unittests/support/affinity_syscall_shim.cpp create mode 100644 libs/qec/unittests/test_pinning_benchmark.cpp diff --git a/libs/qec/lib/decoder.cpp b/libs/qec/lib/decoder.cpp index fe8be0dae..72f5727b9 100644 --- a/libs/qec/lib/decoder.cpp +++ b/libs/qec/lib/decoder.cpp @@ -402,7 +402,18 @@ decoder::get(const std::string &name, const decoder_init &init, node = numa_node_for_cuda_device(dev); CudaDeviceGuard ctor_dev(dev); NumaGuard ctor_numa(node); // preferred mode at construction - auto d = iter->second(init, param_map); + // The affinity knobs above are consumed by the base class. Strip them from + // the map handed to the plugin constructor so a decoder that strictly + // validates its own parameter keys does not reject them. + auto is_affinity_key = [](const std::string &k) { + return k == "cuda_device_id" || k == "numa_node_id" || k == "mempolicy" || + k == "cpu_affinity"; + }; + cudaqx::heterogeneous_map plugin_params; + for (const auto &kv : param_map) + if (!is_affinity_key(kv.first)) + plugin_params.insert(kv.first, kv.second); + auto d = iter->second(init, plugin_params); d->set_hardware_params(param_map); return d; } diff --git a/libs/qec/lib/realtime/qec_realtime_session.cpp b/libs/qec/lib/realtime/qec_realtime_session.cpp index 5245aec3c..97e091bbf 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.cpp +++ b/libs/qec/lib/realtime/qec_realtime_session.cpp @@ -10,7 +10,8 @@ #include "qec_realtime_session.h" -#include "hardware_affinity.h" +// Lib-private header one level up (libs/qec/lib); this file lives in realtime/. +#include "../hardware_affinity.h" #include "cudaq/qec/realtime/decoder_rpc_ids.h" #include "cudaq/qec/realtime/graph_resources.h" #include "cudaq/realtime/daemon/dispatcher/dispatch_kernel_launch.h" diff --git a/libs/qec/unittests/CMakeLists.txt b/libs/qec/unittests/CMakeLists.txt index ea02473b3..9b41b05eb 100644 --- a/libs/qec/unittests/CMakeLists.txt +++ b/libs/qec/unittests/CMakeLists.txt @@ -52,10 +52,25 @@ target_link_libraries(test_decoders_yaml PRIVATE cudaq-qec cudaq-qec-realtime-decoding cudaq-qec-realtime-decoding-simulation - cudaq::cudaq) + cudaq::cudaq + CUDA::cudart) +target_include_directories(test_decoders_yaml PRIVATE ${CUDAToolkit_INCLUDE_DIRS}) add_dependencies(CUDAQXQECUnitTests test_decoders_yaml) gtest_discover_tests(test_decoders_yaml) +# LD_PRELOAD interposer + benchmark/syscall-gate test for hardware pinning. +add_library(affinity_syscall_shim SHARED support/affinity_syscall_shim.cpp) +target_link_libraries(affinity_syscall_shim PRIVATE ${CMAKE_DL_LIBS}) + +add_executable(test_pinning_benchmark test_pinning_benchmark.cpp) +target_link_libraries(test_pinning_benchmark PRIVATE + GTest::gtest_main cudaq-qec cudaq::cudaq CUDA::cudart) +target_include_directories(test_pinning_benchmark PRIVATE ${CUDAToolkit_INCLUDE_DIRS}) +add_dependencies(test_pinning_benchmark affinity_syscall_shim) +add_dependencies(CUDAQXQECUnitTests test_pinning_benchmark) +gtest_discover_tests(test_pinning_benchmark + PROPERTIES ENVIRONMENT "LD_PRELOAD=$") + add_executable(test_device_affinity test_device_affinity.cpp) target_link_libraries(test_device_affinity PRIVATE GTest::gtest_main cudaq-qec cudaq::cudaq CUDA::cudart) target_include_directories(test_device_affinity PRIVATE diff --git a/libs/qec/unittests/support/affinity_syscall_shim.cpp b/libs/qec/unittests/support/affinity_syscall_shim.cpp new file mode 100644 index 000000000..bb5770d85 --- /dev/null +++ b/libs/qec/unittests/support/affinity_syscall_shim.cpp @@ -0,0 +1,31 @@ +// Test-only LD_PRELOAD interposer: counts sched_setaffinity calls so a test can +// assert a bound decode loop issues none. Not linked into the library. +#define _GNU_SOURCE +#include +#include +#include + +namespace { +std::atomic g_count{0}; +} + +extern "C" { + +// Interpose the glibc symbol; forward to the real one via RTLD_NEXT. +int sched_setaffinity(pid_t pid, size_t cpusetsize, const cpu_set_t *mask) { + static int (*real)(pid_t, size_t, const cpu_set_t *) = nullptr; + if (!real) + real = reinterpret_cast( + dlsym(RTLD_NEXT, "sched_setaffinity")); + g_count.fetch_add(1, std::memory_order_relaxed); + return real(pid, cpusetsize, mask); +} + +// Read/reset hooks the test resolves (weakly) from the preloaded shim. +long cudaqx_affinity_syscall_count() { + return g_count.load(std::memory_order_relaxed); +} +void cudaqx_affinity_syscall_reset() { + g_count.store(0, std::memory_order_relaxed); +} +} diff --git a/libs/qec/unittests/test_decoder_pool.cpp b/libs/qec/unittests/test_decoder_pool.cpp index da32ce107..d1758e9c8 100644 --- a/libs/qec/unittests/test_decoder_pool.cpp +++ b/libs/qec/unittests/test_decoder_pool.cpp @@ -90,10 +90,13 @@ TEST(DecoderPool, NvQldpcRunsOnDistinctGpusWhenAvailable) { GTEST_SKIP() << "needs >= 2 GPUs"; std::vector specs; + // nv-qldpc's GPU batched-decode path requires sparse mode. cudaqx::heterogeneous_map o0; o0.insert("cuda_device_id", 0); + o0.insert("use_sparsity", true); cudaqx::heterogeneous_map o1; o1.insert("cuda_device_id", 1); + o1.insert("use_sparsity", true); specs.push_back({0, "nv-qldpc-decoder", H, o0}); specs.push_back({1, "nv-qldpc-decoder", H, o1}); cudaq::qec::decoder_pool pool(std::move(specs)); diff --git a/libs/qec/unittests/test_pinning_benchmark.cpp b/libs/qec/unittests/test_pinning_benchmark.cpp new file mode 100644 index 000000000..66d0df2d3 --- /dev/null +++ b/libs/qec/unittests/test_pinning_benchmark.cpp @@ -0,0 +1,84 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ +#include "cudaq/qec/decoder.h" +#include +#include +#include +#include + +// Resolved from the LD_PRELOAD shim when preloaded; weak so the binary still +// links (and the test cleanly skips) when it is not. +extern "C" __attribute__((weak)) long cudaqx_affinity_syscall_count(); +extern "C" __attribute__((weak)) void cudaqx_affinity_syscall_reset(); + +namespace { +cudaqx::tensor makeH() { + cudaqx::tensor H({2, 3}); + return H; +} +// One decoder with numa_node_id pinned so the decode-time guard's syscalls fire +// on the unbound path. +std::unique_ptr makePinnedLut() { + cudaqx::heterogeneous_map o; + o.insert("numa_node_id", 0); + return cudaq::qec::decoder::get("multi_error_lut", makeH(), o); +} +const std::vector> kChunk = {{0.1, 0.1}, + {0.2, 0.2}}; +} // namespace + +// Deterministic gate: after binding, a decode loop must issue ZERO +// sched_setaffinity calls; the unbound path (canary) must issue > 0 (proving +// the counter is wired and the guard really fires when not bound). +TEST(PinningBenchmark, BoundDecodeLoopIssuesNoAffinitySyscalls) { + if (!cudaqx_affinity_syscall_count) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + auto d = makePinnedLut(); + constexpr int N = 200; + + // Canary: unbound -> per-call guard fires. + cudaqx_affinity_syscall_reset(); + for (int i = 0; i < N; ++i) + d->decode_batch(kChunk); + long unbound = cudaqx_affinity_syscall_count(); + EXPECT_GT(unbound, 0) << "unbound decode loop should issue affinity syscalls " + "(else the gate is vacuous)"; + + // Bound: bind once, then the loop must add none. + d->bind_current_thread(); + cudaqx_affinity_syscall_reset(); + for (int i = 0; i < N; ++i) + d->decode_batch(kChunk); + long bound = cudaqx_affinity_syscall_count(); + EXPECT_EQ(bound, 0) + << "bound decode loop must not re-issue affinity syscalls"; +} + +// Loose A/B (report + gross-regression guard only; timing is noisy). +TEST(PinningBenchmark, BoundThroughputNotWorseThanUnbound) { + auto d = makePinnedLut(); + constexpr int N = 5000; + auto run = [&] { + auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < N; ++i) + d->decode_batch(kChunk); + return std::chrono::duration( + std::chrono::steady_clock::now() - t0) + .count(); + }; + run(); // warm up + double unbound_us = run(); + d->bind_current_thread(); + run(); // warm up bound + double bound_us = run(); + std::printf( + "[pinning A/B] unbound=%.1fus bound=%.1fus (%d decodes) ratio=%.2f\n", + unbound_us, bound_us, N, bound_us / unbound_us); + // Pinning removes per-call guard work; it must not be materially slower. + EXPECT_LT(bound_us, unbound_us * 1.5); +} From 765bc126c3a97c005dd16eeb2972686fab0dc69b Mon Sep 17 00:00:00 2001 From: kvmto Date: Thu, 2 Jul 2026 12:34:24 +0000 Subject: [PATCH 07/16] test(qec): skip mempolicy assertions where the syscalls are blocked Container runtimes commonly deny set_mempolicy/get_mempolicy (seccomp without CAP_SYS_NICE). The library already degrades gracefully there; the three tests that assert the syscalls' effects now probe first and skip instead of failing. Verified both ways: on a permissive host all 15 tests still run and pass; with the mempolicy family blocked the three skip and the rest pass. Signed-off-by: kvmto --- libs/qec/unittests/test_device_affinity.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/libs/qec/unittests/test_device_affinity.cpp b/libs/qec/unittests/test_device_affinity.cpp index a899d6a11..34d46b9ba 100644 --- a/libs/qec/unittests/test_device_affinity.cpp +++ b/libs/qec/unittests/test_device_affinity.cpp @@ -51,14 +51,28 @@ TEST(DeviceAffinity, ReadNegativeThrows) { } #if defined(__linux__) +// Container runtimes commonly block the mempolicy syscalls (seccomp without +// CAP_SYS_NICE). The library degrades gracefully there (warns, keeps going), +// but tests that assert the syscalls' effects must skip instead of fail. +static bool mempolicy_syscalls_usable() { + int mode = -1; + if (syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL) != 0) + return false; + return syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL) == 0; +} + TEST(HardwareAffinity, MempolicyDefaultIsPreferredNotBind) { namespace da = cudaq::qec::detail_affinity; + if (!mempolicy_syscalls_usable()) + GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; da::bind_this_thread_to_numa_node(0); EXPECT_EQ(da::current_thread_mempolicy_mode(), MPOL_PREFERRED); syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); } TEST(HardwareAffinity, MempolicyBindWhenRequested) { namespace da = cudaq::qec::detail_affinity; + if (!mempolicy_syscalls_usable()) + GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; da::bind_this_thread_to_numa_node(0, da::mempolicy_mode::bind); EXPECT_EQ(da::current_thread_mempolicy_mode(), MPOL_BIND); syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); @@ -120,6 +134,8 @@ TEST(HardwareAffinity, BindThreadPinsAffinityToNodeCpus) { } TEST(HardwareAffinity, NegativeNodeIsNoop) { namespace da = cudaq::qec::detail_affinity; + if (!mempolicy_syscalls_usable()) + GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; cpu_set_t before; CPU_ZERO(&before); sched_getaffinity(0, sizeof(before), &before); From dca1f0e4901bdb608a8d2cab5cd683e55bbe2286 Mon Sep 17 00:00:00 2001 From: kvmto Date: Fri, 3 Jul 2026 13:32:57 +0000 Subject: [PATCH 08/16] fix(qec): harden hardware-pinning guards and drop decoder_pool Remove decoder_pool: out of scope for this PR. Two concurrently constructed decoders cover the multi-GPU composition directly (TwoTrtDecodersConcurrently). Guard correctness: - roll back mempolicy, CPU affinity, and CUDA device when bind_current_thread() fails partway; never leave a half-bound thread - decode_on_pinned_thread() restores the caller's binding after the one-shot worker exits - temporary guards never apply a mempolicy they cannot restore (bind_this_thread_to_numa_node gains an apply_mempolicy switch) - NUMA-bind failures in realtime session threads warn instead of terminating the process - trt_decoder::decode_batch() honors bind_current_thread() via is_bound_here() (now protected) instead of re-binding every call - device-restore failures in guard destructors warn instead of being silently ignored Tests: - consolidate three probe decoders into one placement_probe_decoder - LD_PRELOAD shim counts all four placement syscalls and can inject get_mempolicy failures; test_trt_decoder now runs under it - invariant coverage: every entry point x {bound, unbound} asserts syscall counts and exact placement restoration; error-path rollback and bind/pinned-thread interaction tests added Signed-off-by: kvmto --- libs/qec/include/cudaq/qec/decoder.h | 30 +- libs/qec/include/cudaq/qec/decoder_pool.h | 60 --- libs/qec/lib/CMakeLists.txt | 1 - libs/qec/lib/decoder.cpp | 216 ++++++---- libs/qec/lib/decoder_pool.cpp | 115 ------ .../plugins/trt_decoder/CMakeLists.txt | 1 + .../plugins/trt_decoder/trt_decoder.cpp | 78 +++- libs/qec/lib/hardware_affinity.h | 80 +++- .../qec/lib/realtime/qec_realtime_session.cpp | 28 +- libs/qec/unittests/CMakeLists.txt | 11 +- .../pymatching/test_pymatching_realtime.cpp | 18 + .../qec/unittests/decoders/sample_decoder.cpp | 99 ++--- .../decoders/trt_decoder/test_trt_decoder.cpp | 384 +++++++++++++++--- .../support/affinity_syscall_shim.cpp | 110 ++++- libs/qec/unittests/support/thread_placement.h | 79 ++++ libs/qec/unittests/test_decoder_pool.cpp | 199 --------- libs/qec/unittests/test_decoders.cpp | 238 +++++++++-- libs/qec/unittests/test_device_affinity.cpp | 13 +- libs/qec/unittests/test_pinning_benchmark.cpp | 230 +++++++++++ 19 files changed, 1330 insertions(+), 660 deletions(-) delete mode 100644 libs/qec/include/cudaq/qec/decoder_pool.h delete mode 100644 libs/qec/lib/decoder_pool.cpp create mode 100644 libs/qec/unittests/support/thread_placement.h delete mode 100644 libs/qec/unittests/test_decoder_pool.cpp diff --git a/libs/qec/include/cudaq/qec/decoder.h b/libs/qec/include/cudaq/qec/decoder.h index f2ee953c9..50d65e4fc 100644 --- a/libs/qec/include/cudaq/qec/decoder.h +++ b/libs/qec/include/cudaq/qec/decoder.h @@ -15,12 +15,14 @@ #include "cudaq/qec/detector_error_model.h" #include "cudaq/qec/device_affinity.h" #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -174,6 +176,13 @@ class decoder /// length of the syndrome vector should be equal to `syndrome_size`. /// @returns Vector of length `block_size` with soft probabilities of errors /// in each index. + /// Note: this entry point has no automatic CUDA/NUMA guard. It is a pure + /// virtual that plugin implementations override directly, so the base + /// class cannot wrap it without an ABI-breaking vtable change. + /// decode_batch(), decode_async(), decode(tensor), and enqueue_syndrome() + /// all apply the guard around their own call into this function; a caller + /// invoking this overload directly on an unbound decoder is responsible + /// for its own placement (e.g. call bind_current_thread() first). virtual decoder_result decode(const std::vector &syndrome) = 0; /// @brief Decode a single syndrome @@ -386,24 +395,25 @@ class decoder int cuda_device_id_ = -1; /// Target NUMA node for this decoder. -1 = no binding. int numa_node_id_ = -1; - /// Set once bind_current_thread() has pinned the owning thread, so the - /// synchronous decode_batch() guard can skip its redundant set/restore. - bool bound_persistently_ = false; + /// Thread that bind_current_thread() pinned, if any. Guarded call sites + /// (decode_batch, decode(tensor), enqueue_syndrome) skip their per-call + /// CUDA/NUMA guard only when the calling thread matches. A + /// default-constructed id means no thread is bound. + std::atomic bound_thread_{}; /// NUMA memory-policy mode for this decoder (default soft PREFERRED). cudaq::qec::mempolicy_mode mempolicy_ = cudaq::qec::mempolicy_mode::preferred; /// Explicit CPU cores for this decoder's owning thread (empty = use node /// cpuset). std::vector cpu_affinity_; - /// Set once the current-device mismatch warning has fired, so it prints at - /// most once per decoder instance. - bool device_mismatch_warned_ = false; + + /// True if bind_current_thread() was called, and the CURRENT thread is the + /// one that called it -- the only thread allowed to skip the per-call + /// CUDA/NUMA guard. Protected so plugin overrides of decode_batch() can + /// apply the same skip as the base-class entry points. + bool is_bound_here() const; private: decode_result_type result_type_ = decode_result_type::decode_to_errs; - - /// Warn (once) if this decoder has an explicit cuda_device_id and the - /// calling thread's current CUDA device does not match it. - void warn_if_device_mismatch(); }; /// @brief Convert a single soft probability to a hard 0/1 decision. diff --git a/libs/qec/include/cudaq/qec/decoder_pool.h b/libs/qec/include/cudaq/qec/decoder_pool.h deleted file mode 100644 index 6308d1df3..000000000 --- a/libs/qec/include/cudaq/qec/decoder_pool.h +++ /dev/null @@ -1,60 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ -#pragma once - -#include "cudaq/qec/decoder.h" -#include -#include -#include -#include -#include - -namespace cudaq::qec { - -/// @brief One decoder's placement + construction inputs for a decoder_pool. -struct pool_decoder_spec { - int id; ///< Caller-chosen routing id. - std::string name; ///< Registered decoder name. - cudaqx::tensor H; ///< Parity-check matrix. - cudaqx::heterogeneous_map options; ///< get() options (cuda_device_id, ...). -}; - -/// @brief Runs a set of decoders concurrently, each on its own worker thread -/// bound (bind_current_thread) to that decoder's CUDA device / NUMA node. The -/// decoder is constructed on its worker thread so its resources land on-node. -class decoder_pool { -public: - explicit decoder_pool(std::vector specs); - ~decoder_pool(); - decoder_pool(const decoder_pool &) = delete; - decoder_pool &operator=(const decoder_pool &) = delete; - - /// @brief Decode every id's syndromes on that id's pinned worker, with all - /// decoders running concurrently; blocks until they finish and returns the - /// results grouped by id. Each id maps to that decoder's syndromes (the - /// per-decoder batched decode happens inside its worker). Throws if an id has - /// no matching decoder, or rethrows a worker's decode exception. - std::unordered_map> decode_all( - const std::unordered_map>> &work); - - /// @brief Non-blocking streaming submit: enqueue `syndromes` on `id`'s pinned - /// worker and return immediately with a future for that chunk's results. - /// Submit chunks over time and consume the futures as they resolve to stream - /// per id, concurrently; a size-1 chunk streams a single syndrome. Throws if - /// `id` has no matching decoder; the future rethrows a worker decode - /// exception. - std::future> - submit(int id, std::vector> syndromes); - -private: - struct worker; - std::vector> workers_; - std::unordered_map by_id_; -}; - -} // namespace cudaq::qec diff --git a/libs/qec/lib/CMakeLists.txt b/libs/qec/lib/CMakeLists.txt index 707ce06ae..c47d8b840 100644 --- a/libs/qec/lib/CMakeLists.txt +++ b/libs/qec/lib/CMakeLists.txt @@ -41,7 +41,6 @@ endif() set(QEC_SOURCES code.cpp decoder.cpp - decoder_pool.cpp detector_error_model.cpp experiments.cpp logger.cpp diff --git a/libs/qec/lib/decoder.cpp b/libs/qec/lib/decoder.cpp index 4b9a31246..6c535d2b3 100644 --- a/libs/qec/lib/decoder.cpp +++ b/libs/qec/lib/decoder.cpp @@ -34,15 +34,10 @@ namespace { -inline cudaq::qec::detail_affinity::mempolicy_mode -to_affinity_mode(cudaq::qec::mempolicy_mode m) { - return m == cudaq::qec::mempolicy_mode::bind - ? cudaq::qec::detail_affinity::mempolicy_mode::bind - : cudaq::qec::detail_affinity::mempolicy_mode::preferred; -} - // RAII: sets the calling thread's CUDA current device, restores on destruction. -// target < 0 = no-op. Throws std::runtime_error if target is out of range. +// target < 0 = no-op. Throws std::runtime_error if: target is out of range +// of the visible device count, cudaGetDevice() fails while checking the +// current device, or cudaSetDevice() fails while switching to target. struct CudaDeviceGuard { int prev_ = -1; bool active_ = false; @@ -56,9 +51,13 @@ struct CudaDeviceGuard { "cuda_device_id " + std::to_string(target) + " out of range (device_count=" + std::to_string(count) + ")"); if (cudaGetDevice(&prev_) != cudaSuccess) { - prev_ = -1; + // Can't determine the current device; warn and skip the switch so the + // decode proceeds on whatever device the caller is already on. + cudaq::qec::detail_affinity::affinity_warn( + "CudaDeviceGuard: cudaGetDevice failed for cuda_device_id " + + std::to_string(target) + "; skipping device switch"); return; - } // can't safely switch + } if (prev_ != target) { cudaError_t e = cudaSetDevice(target); if (e != cudaSuccess) @@ -69,8 +68,12 @@ struct CudaDeviceGuard { } } ~CudaDeviceGuard() { - if (active_) - cudaSetDevice(prev_); + if (!active_) + return; + if (cudaSetDevice(prev_) != cudaSuccess) + cudaq::qec::detail_affinity::affinity_warn( + "CudaDeviceGuard: failed to restore prior CUDA device " + + std::to_string(prev_) + "; thread may remain on wrong device"); } CudaDeviceGuard(const CudaDeviceGuard &) = delete; CudaDeviceGuard &operator=(const CudaDeviceGuard &) = delete; @@ -85,26 +88,36 @@ struct NumaGuard { bool has_prev_affinity_ = false; bool mempol_set_ = false; cpu_set_t prev_set_{}; + cudaq::qec::detail_affinity::mempolicy_state prev_mempolicy_; - explicit NumaGuard( - int node, cudaq::qec::detail_affinity::mempolicy_mode mode = - cudaq::qec::detail_affinity::mempolicy_mode::preferred) { + explicit NumaGuard(int node, cudaq::qec::mempolicy_mode mode = + cudaq::qec::mempolicy_mode::preferred) { if (node < 0) return; CPU_ZERO(&prev_set_); has_prev_affinity_ = (sched_getaffinity(0, sizeof(prev_set_), &prev_set_) == 0); + if (node < static_cast(sizeof(unsigned long) * 8)) { + prev_mempolicy_ = cudaq::qec::detail_affinity::capture_thread_mempolicy(); + mempol_set_ = (prev_mempolicy_.mode >= 0); + if (!mempol_set_) + cudaq::qec::detail_affinity::affinity_warn( + "NumaGuard: prior thread mempolicy unreadable (get_mempolicy " + "failed); temporary NUMA memory policy skipped for this decode"); + } + // A temporary guard must never apply a policy it cannot restore: skip the + // mempolicy half when the capture failed (affinity is still applied, its + // restore path is independent). + cudaq::qec::detail_affinity::bind_this_thread_to_numa_node( + node, mode, /*apply_mempolicy=*/mempol_set_); + // Arm the affinity restore only after a successful bind; if bind throws the + // affinity was not changed and there is nothing to undo. affinity_set_ = has_prev_affinity_; - mempol_set_ = (node < static_cast(sizeof(unsigned long) * 8)); - cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(node, mode); } ~NumaGuard() { if (mempol_set_) - if (syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL) != 0) - cudaq::qec::detail_affinity::affinity_warn( - "NumaGuard restore: failed to reset thread mempolicy: " + - std::string(std::strerror(errno)) + "; thread may remain bound"); + cudaq::qec::detail_affinity::restore_thread_mempolicy(prev_mempolicy_); if (affinity_set_ && has_prev_affinity_) if (sched_setaffinity(0, sizeof(prev_set_), &prev_set_) != 0) cudaq::qec::detail_affinity::affinity_warn( @@ -117,18 +130,12 @@ struct NumaGuard { }; #else struct NumaGuard { - explicit NumaGuard( - int node, cudaq::qec::detail_affinity::mempolicy_mode mode = - cudaq::qec::detail_affinity::mempolicy_mode::preferred) { + explicit NumaGuard(int node, cudaq::qec::mempolicy_mode mode = + cudaq::qec::mempolicy_mode::preferred) { (void)mode; if (node < 0) return; - static bool warned = false; - if (!warned) { - std::cerr << "[cudaq-qec] numa_node_id ignored: NUMA binding is only " - "supported on Linux.\n"; - warned = true; - } + cudaq::qec::detail_affinity::warn_numa_unsupported_once(); } }; #endif @@ -248,6 +255,8 @@ void decoder::set_hardware_params(const cudaqx::heterogeneous_map ¶ms) { } int decoder::bind_current_thread() { + int prev_dev = -1; + bool dev_switched = false; if (cuda_device_id_ >= 0) { // Persistent set on this thread (CudaDeviceGuard restores on scope exit, so // set directly here and let it stick). @@ -256,47 +265,87 @@ int decoder::bind_current_thread() { throw std::runtime_error( "cuda_device_id " + std::to_string(cuda_device_id_) + " out of range or CUDA unavailable in bind_current_thread"); + if (cudaGetDevice(&prev_dev) != cudaSuccess) + prev_dev = -1; cudaError_t e = cudaSetDevice(cuda_device_id_); if (e != cudaSuccess) throw std::runtime_error("bind_current_thread: cudaSetDevice(" + std::to_string(cuda_device_id_) + ") failed: " + cudaGetErrorString(e)); + dev_switched = (prev_dev >= 0 && prev_dev != cuda_device_id_); } - cudaq::qec::detail_affinity::bind_this_thread_to_numa_node( - numa_node_id_, to_affinity_mode(mempolicy_)); - bound_persistently_ = true; - if (numa_node_id_ >= 0) { - // Best-effort confirmation: warn if the calling thread is not actually - // running on the requested node's CPUs (e.g. locked cpuset in a container). + // Capture thread placement before the NUMA bind so every side effect can be + // rolled back if any later step throws and bound_thread_ is never stored. + auto prev_mempol_bind = + cudaq::qec::detail_affinity::capture_thread_mempolicy(); #if defined(__linux__) - cpu_set_t want, have; - CPU_ZERO(&want); - CPU_ZERO(&have); - if (cudaq::qec::detail_affinity::build_node_cpuset(numa_node_id_, want) && - sched_getaffinity(0, sizeof(have), &have) == 0) { - bool on_node = false; - for (int c = 0; c < CPU_SETSIZE; ++c) - if (CPU_ISSET(c, &have) && CPU_ISSET(c, &want)) { - on_node = true; - break; - } - if (!on_node) - cudaq::qec::detail_affinity::affinity_warn( - "bind_current_thread: numa_node_id " + - std::to_string(numa_node_id_) + - " requested but the calling thread could not be pinned to it"); + cpu_set_t prev_affinity; + CPU_ZERO(&prev_affinity); + const bool has_prev_affinity = + (sched_getaffinity(0, sizeof(prev_affinity), &prev_affinity) == 0); +#endif + try { + cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(numa_node_id_, + mempolicy_); + if (numa_node_id_ >= 0) { + // Best-effort confirmation: warn if the calling thread is not actually + // running on the requested node's CPUs (e.g. locked cpuset in a + // container). +#if defined(__linux__) + cpu_set_t want, have; + CPU_ZERO(&want); + CPU_ZERO(&have); + if (cudaq::qec::detail_affinity::build_node_cpuset(numa_node_id_, want) && + sched_getaffinity(0, sizeof(have), &have) == 0) { + bool on_node = false; + for (int c = 0; c < CPU_SETSIZE; ++c) + if (CPU_ISSET(c, &have) && CPU_ISSET(c, &want)) { + on_node = true; + break; + } + if (!on_node) + cudaq::qec::detail_affinity::affinity_warn( + "bind_current_thread: numa_node_id " + + std::to_string(numa_node_id_) + + " requested but the calling thread could not be pinned to it"); + } +#endif } + if (!cpu_affinity_.empty()) + cudaq::qec::detail_affinity::set_thread_cpu_affinity(cpu_affinity_); + } catch (...) { + // Roll back every side effect applied above so a failed bind leaves the + // thread exactly as it was: mempolicy, CPU affinity, and CUDA device. + cudaq::qec::detail_affinity::restore_thread_mempolicy(prev_mempol_bind); +#if defined(__linux__) + if (has_prev_affinity) + sched_setaffinity(0, sizeof(prev_affinity), &prev_affinity); #endif + if (dev_switched && cudaSetDevice(prev_dev) != cudaSuccess) + cudaq::qec::detail_affinity::affinity_warn( + "bind_current_thread: failed to restore prior CUDA device " + + std::to_string(prev_dev) + " after a failed bind"); + throw; } - if (!cpu_affinity_.empty()) - cudaq::qec::detail_affinity::set_thread_cpu_affinity(cpu_affinity_); + // Only mark this thread as bound once every step that can throw has + // succeeded, so a failed bind never leaves the decoder falsely marked as + // bound to this thread. + bound_thread_.store(std::this_thread::get_id(), std::memory_order_release); return numa_node_id_; } +bool decoder::is_bound_here() const { + return bound_thread_.load(std::memory_order_acquire) == + std::this_thread::get_id(); +} + decoder_result decoder::decode_on_pinned_thread(const std::vector &syndrome) { decoder_result result; std::exception_ptr err; + // Snapshot the caller's binding before the worker overwrites it, so we can + // restore it after join. + const auto caller_id = bound_thread_.load(std::memory_order_acquire); std::thread worker([&] { try { bind_current_thread(); @@ -304,31 +353,24 @@ decoder::decode_on_pinned_thread(const std::vector &syndrome) { } catch (...) { err = std::current_exception(); } + // Clear the one-shot worker's own binding on both success and exception + // paths so a recycled thread id never spuriously skips the guards. + auto me = std::this_thread::get_id(); + bound_thread_.compare_exchange_strong(me, std::thread::id{}, + std::memory_order_acq_rel); }); worker.join(); + // Restore the caller's prior binding that the worker overwrote. The worker + // has joined so this store races with nothing. + bound_thread_.store(caller_id, std::memory_order_release); if (err) - std::rethrow_exception(err); // surface an OOB-device throw to the caller + std::rethrow_exception(err); return result; } -void decoder::warn_if_device_mismatch() { - if (cuda_device_id_ < 0 || device_mismatch_warned_) - return; - int current = -1; - if (cudaGetDevice(¤t) == cudaSuccess && current != cuda_device_id_) { - cudaq::qec::detail_affinity::affinity_warn( - "decoder pinned to cuda_device_id " + std::to_string(cuda_device_id_) + - " but decoding on current device " + std::to_string(current) + - "; call bind_current_thread() on this thread, or use " - "decode_batch/decode_async"); - device_mismatch_warned_ = true; - } -} - // Provide a trivial implementation of for tensor decode call. Child // classes should override this if they never want to pass through floats. decoder_result decoder::decode(const cudaqx::tensor &syndrome) { - warn_if_device_mismatch(); // Check tensor is of order-1 // If order >1, we could check that other modes are of dim = 1 such that // n x 1, or 1 x n tensors are still valid. @@ -339,6 +381,9 @@ decoder_result decoder::decode(const cudaqx::tensor &syndrome) { std::vector vec_cast(syndrome.data(), syndrome.data() + syndrome.shape()[0]); convert_vec_hard_to_soft(vec_cast, soft_syndrome); + const bool skip_guard = is_bound_here(); + CudaDeviceGuard dev(skip_guard ? -1 : cuda_device_id_); + NumaGuard numa(skip_guard ? -1 : numa_node_id_, mempolicy_); return decode(soft_syndrome); } @@ -348,11 +393,12 @@ std::vector decoder::decode_batch(const std::vector> &syndrome) { // Apply affinity once for the whole batch (not per-syndrome) to avoid // repeated syscall overhead on the hot path. - // If the caller already bound this thread via bind_current_thread(), the - // per-call guards are redundant set/restore syscalls — skip them. - CudaDeviceGuard dev(bound_persistently_ ? -1 : cuda_device_id_); - NumaGuard numa(bound_persistently_ ? -1 : numa_node_id_, - to_affinity_mode(mempolicy_)); + // Skip the guard only on the exact thread that called bind_current_thread() + // -- a different thread must still be guarded even if this decoder was + // bound elsewhere. + const bool skip_guard = is_bound_here(); + CudaDeviceGuard dev(skip_guard ? -1 : cuda_device_id_); + NumaGuard numa(skip_guard ? -1 : numa_node_id_, mempolicy_); std::vector result; result.reserve(syndrome.size()); for (auto &s : syndrome) @@ -377,7 +423,7 @@ decoder::decode_async(const std::vector &syndrome) { return std::async(std::launch::async, [this, syndrome, cuda_id, numa_id, mempolicy] { CudaDeviceGuard dev(cuda_id); - NumaGuard numa(numa_id, to_affinity_mode(mempolicy)); + NumaGuard numa(numa_id, mempolicy); return this->decode(syndrome); }); } @@ -394,14 +440,17 @@ decoder::get(const std::string &name, const decoder_init &init, ". Run with CUDAQ_LOG_LEVEL=info (environment variable) to see " "additional plugin diagnostics at startup."); // Guards during construction so allocations land on the right hardware. - // Restored before this function returns; decode-time affinity is re-applied - // per call in decode_batch() and decode_async(). + // Restored before this function returns; decode-time affinity is + // re-applied per call at every guarded entry point: decode_batch(), + // decode_async(), decode(const cudaqx::tensor &), and + // enqueue_syndrome(). const int dev = cudaq::qec::read_cuda_device_id(param_map); int node = cudaq::qec::read_numa_node_id(param_map); if (node < 0 && dev >= 0) node = numa_node_for_cuda_device(dev); + const auto ctor_mempolicy = cudaq::qec::read_mempolicy(param_map); CudaDeviceGuard ctor_dev(dev); - NumaGuard ctor_numa(node); // preferred mode at construction + NumaGuard ctor_numa(node, ctor_mempolicy); // The affinity knobs above are consumed by the base class. Strip them from // the map handed to the plugin constructor so a decoder that strictly // validates its own parameter keys does not reject them. @@ -633,8 +682,13 @@ bool decoder::enqueue_syndrome(const uint8_t *syndrome, // Send the data to the decoder. convert_vec_hard_to_soft(pimpl->persistent_detector_buffer, pimpl->persistent_soft_detector_buffer); - warn_if_device_mismatch(); - auto decoded_result = decode(pimpl->persistent_soft_detector_buffer); + decoder_result decoded_result; + { + const bool skip_guard = is_bound_here(); + CudaDeviceGuard dev(skip_guard ? -1 : cuda_device_id_); + NumaGuard numa(skip_guard ? -1 : numa_node_id_, mempolicy_); + decoded_result = decode(pimpl->persistent_soft_detector_buffer); + } // If we didn't get a decoded result, just return if (pimpl->is_sliding_window) { diff --git a/libs/qec/lib/decoder_pool.cpp b/libs/qec/lib/decoder_pool.cpp deleted file mode 100644 index e3260282a..000000000 --- a/libs/qec/lib/decoder_pool.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ -#include "cudaq/qec/decoder_pool.h" - -#include -#include -#include -#include -#include -#include - -namespace cudaq::qec { - -// One persistent, GPU/NUMA-pinned worker owning a single decoder and a job -// queue. The decoder is constructed and bound on this worker's own thread. -struct decoder_pool::worker { - struct job { - std::vector> syndromes; // owned copy (crosses threads) - std::promise> result; - }; - - pool_decoder_spec spec; - std::mutex m; - std::condition_variable cv; - bool stop = false; - std::queue> jobs; - std::thread thread; - - explicit worker(pool_decoder_spec s) : spec(std::move(s)) { - thread = std::thread([this] { run(); }); - } - ~worker() { - { - std::lock_guard lk(m); - stop = true; - } - cv.notify_all(); - if (thread.joinable()) - thread.join(); - } - - std::future> - submit(std::vector> syndromes) { - auto j = std::make_unique(); - j->syndromes = std::move(syndromes); - auto fut = j->result.get_future(); - { - std::lock_guard lk(m); - jobs.push(std::move(j)); - } - cv.notify_one(); - return fut; - } - - void run() { - // Construct + pin on this worker thread so resources land on the target - // GPU. - auto dec = decoder::get(spec.name, spec.H, spec.options); - dec->bind_current_thread(); - for (;;) { - std::unique_ptr j; - { - std::unique_lock lk(m); - cv.wait(lk, [this] { return stop || !jobs.empty(); }); - if (stop && jobs.empty()) - return; - j = std::move(jobs.front()); - jobs.pop(); - } - try { - j->result.set_value(dec->decode_batch(j->syndromes)); - } catch (...) { - j->result.set_exception(std::current_exception()); - } - } - } -}; - -decoder_pool::decoder_pool(std::vector specs) { - workers_.reserve(specs.size()); - for (auto &s : specs) { - int id = s.id; - workers_.push_back(std::make_unique(std::move(s))); - by_id_[id] = workers_.back().get(); - } -} - -decoder_pool::~decoder_pool() = default; // worker dtors stop + join - -std::future> -decoder_pool::submit(int id, std::vector> syndromes) { - auto it = by_id_.find(id); - if (it == by_id_.end()) - throw std::runtime_error("decoder_pool::submit: no decoder with id " + - std::to_string(id)); - return it->second->submit(std::move(syndromes)); -} - -std::unordered_map> decoder_pool::decode_all( - const std::unordered_map>> &work) { - std::unordered_map>> futures; - for (const auto &[id, syndromes] : work) - futures.emplace(id, submit(id, syndromes)); // copies chunk into the worker - std::unordered_map> results; - for (auto &[id, fut] : futures) - results.emplace(id, fut.get()); // rethrows any worker exception - return results; -} - -} // namespace cudaq::qec diff --git a/libs/qec/lib/decoders/plugins/trt_decoder/CMakeLists.txt b/libs/qec/lib/decoders/plugins/trt_decoder/CMakeLists.txt index 84a5a7b76..82603b0d0 100644 --- a/libs/qec/lib/decoders/plugins/trt_decoder/CMakeLists.txt +++ b/libs/qec/lib/decoders/plugins/trt_decoder/CMakeLists.txt @@ -124,6 +124,7 @@ if(CUDAQ_QEC_TRT_DECODER_ENABLED) ${CMAKE_SOURCE_DIR}/libs/qec/include ${CMAKE_SOURCE_DIR}/libs/core/include PRIVATE + ${CMAKE_SOURCE_DIR}/libs/qec/lib ${TENSORRT_INCLUDE_DIR} ${CUDAToolkit_INCLUDE_DIRS} ) diff --git a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp index 1a9a99bce..9173f8c30 100644 --- a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp +++ b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp @@ -19,6 +19,10 @@ #include #include #include +#if defined(__linux__) +#include +#endif +#include "hardware_affinity.h" // TensorRT headers #include "NvInfer.h" @@ -888,14 +892,28 @@ decoder_result trt_decoder::decode(const std::vector &syndrome) { std::vector trt_decoder::decode_batch(const std::vector> &syndromes) { - // This override bypasses decoder::decode_batch()'s CudaDeviceGuard; apply it - // here so TRT inference lands on the right device regardless of the caller. + // Skip device/NUMA guards when the calling thread already called + // bind_current_thread() — mirrors the base class decode_batch() behaviour. + const bool skip_guard = is_bound_here(); + + // This override bypasses decoder::decode_batch()'s CudaDeviceGuard/NumaGuard + // (private to decoder.cpp, not reachable across this plugin's shared-library + // boundary); reimplement the same contract here so TRT inference lands on + // the right device/NUMA node regardless of the caller. int _prev_dev = -1; bool _dev_switched = false; - if (cuda_device_id_ >= 0) { - cudaGetDevice(&_prev_dev); + if (!skip_guard && cuda_device_id_ >= 0) { + cudaError_t _get_err = cudaGetDevice(&_prev_dev); + if (_get_err != cudaSuccess) + throw std::runtime_error( + std::string("trt_decoder::decode_batch: cudaGetDevice failed: ") + + cudaGetErrorString(_get_err)); if (_prev_dev != cuda_device_id_) { - cudaSetDevice(cuda_device_id_); + cudaError_t _set_err = cudaSetDevice(cuda_device_id_); + if (_set_err != cudaSuccess) + throw std::runtime_error("trt_decoder::decode_batch: cudaSetDevice(" + + std::to_string(cuda_device_id_) + + ") failed: " + cudaGetErrorString(_set_err)); _dev_switched = true; } } @@ -903,11 +921,57 @@ trt_decoder::decode_batch(const std::vector> &syndromes) { int prev; bool active; ~_RestoreDevice() { - if (active) - cudaSetDevice(prev); + if (!active) + return; + if (cudaSetDevice(prev) != cudaSuccess) + cudaq::qec::detail_affinity::affinity_warn( + "trt_decoder: failed to restore prior CUDA device " + + std::to_string(prev) + "; thread may remain on wrong device"); } } _dev_guard{_prev_dev, _dev_switched}; + namespace da = cudaq::qec::detail_affinity; + struct _ScopedNuma { + bool active = false; + bool has_prev_affinity = false; +#if defined(__linux__) + cpu_set_t prev_set{}; +#endif + da::mempolicy_state prev_mempolicy; + _ScopedNuma(int node, cudaq::qec::mempolicy_mode mode) { + if (node < 0) + return; + bool apply_mempol = true; +#if defined(__linux__) + has_prev_affinity = + (sched_getaffinity(0, sizeof(prev_set), &prev_set) == 0); + prev_mempolicy = da::capture_thread_mempolicy(); + // A temporary guard must never apply a policy it cannot restore. + apply_mempol = (prev_mempolicy.mode >= 0); + if (!apply_mempol) + da::affinity_warn( + "trt_decoder: prior thread mempolicy unreadable (get_mempolicy " + "failed); temporary NUMA memory policy skipped for this decode"); +#endif + da::bind_this_thread_to_numa_node(node, mode, apply_mempol); + // Arm the restore only after a successful bind; bind throws before any + // syscall for out-of-range nodes, so there is nothing to undo then. + active = true; + } + ~_ScopedNuma() { + if (!active) + return; +#if defined(__linux__) + da::restore_thread_mempolicy(prev_mempolicy); + if (has_prev_affinity && + sched_setaffinity(0, sizeof(prev_set), &prev_set) != 0) + da::affinity_warn( + "trt_decoder: failed to reset thread affinity after decode; " + "thread may remain pinned"); +#endif + } + } _numa_guard(skip_guard ? -1 : numa_node_id_, mempolicy_); + // Validate that we have syndromes to decode if (syndromes.empty()) { return {}; diff --git a/libs/qec/lib/hardware_affinity.h b/libs/qec/lib/hardware_affinity.h index ecdbc6c83..ad5fa1842 100644 --- a/libs/qec/lib/hardware_affinity.h +++ b/libs/qec/lib/hardware_affinity.h @@ -10,6 +10,7 @@ // decoder and the realtime session. Never include from a public header. #pragma once +#include "cudaq/qec/device_affinity.h" #include #include #include @@ -46,9 +47,8 @@ inline void affinity_info(const std::string &msg) { #if defined(__linux__) -enum class mempolicy_mode { preferred, bind }; -inline int mempolicy_syscall_mode(mempolicy_mode m) { - return (m == mempolicy_mode::bind) ? MPOL_BIND : MPOL_PREFERRED; +inline int mempolicy_syscall_mode(cudaq::qec::mempolicy_mode m) { + return (m == cudaq::qec::mempolicy_mode::bind) ? MPOL_BIND : MPOL_PREFERRED; } // Parse /sys/devices/system/node/node/cpulist e.g. "0-7,16-23". @@ -83,11 +83,15 @@ inline bool build_node_cpuset(int node, cpu_set_t &out) { // Persistently bind the CALLING thread's CPU affinity + memory policy to a NUMA // node. No restore. node < 0 = no-op. Only pins CPU affinity when prior // affinity is readable (avoids permanent pinning in a locked-cpuset container). +// apply_mempolicy=false skips the set_mempolicy half; temporary guards use it +// when the prior policy could not be captured, so they never apply a policy +// they cannot restore (the same only-if-restorable rule as the affinity half). // Throws if node cannot be encoded in the mempolicy nodemask (node >= 64). // Warns (does not throw) if the OS declines an otherwise well-formed request. -inline void -bind_this_thread_to_numa_node(int node, - mempolicy_mode mode = mempolicy_mode::preferred) { +inline void bind_this_thread_to_numa_node( + int node, + cudaq::qec::mempolicy_mode mode = cudaq::qec::mempolicy_mode::preferred, + bool apply_mempolicy = true) { if (node < 0) return; if (node >= static_cast(sizeof(unsigned long) * 8)) @@ -111,6 +115,8 @@ bind_this_thread_to_numa_node(int node, " requested but its CPU list could not be read (or prior " "affinity unreadable); CPU affinity left unpinned"); } + if (!apply_mempolicy) + return; unsigned long nodemask = 1UL << node; if (syscall(SYS_set_mempolicy, mempolicy_syscall_mode(mode), &nodemask, static_cast(sizeof(nodemask) * 8)) != 0) @@ -124,9 +130,9 @@ bind_this_thread_to_numa_node(int node, // to relocate the pages. node < 0 / null / 0 bytes = no-op. // Throws if node cannot be encoded in the mempolicy nodemask (node >= 64). // Warns (does not throw) if the OS declines an otherwise well-formed request. -inline void -bind_region_to_numa_node(void *p, std::size_t bytes, int node, - mempolicy_mode mode = mempolicy_mode::preferred) { +inline void bind_region_to_numa_node( + void *p, std::size_t bytes, int node, + cudaq::qec::mempolicy_mode mode = cudaq::qec::mempolicy_mode::preferred) { if (node < 0 || p == nullptr || bytes == 0) return; if (node >= static_cast(sizeof(unsigned long) * 8)) @@ -152,6 +158,36 @@ inline int current_thread_mempolicy_mode() { return mode; } +// Full state needed to restore a thread's memory policy exactly as it was. +// mode == -1 means the capture failed (nothing to restore). +struct mempolicy_state { + int mode = -1; + unsigned long nodemask[16] = {0}; // up to 1024 nodes +}; + +// Capture the calling thread's current memory policy (mode + nodemask) so it +// can be restored exactly later. +inline mempolicy_state capture_thread_mempolicy() { + mempolicy_state s; + if (syscall(SYS_get_mempolicy, &s.mode, s.nodemask, + static_cast(sizeof(s.nodemask) * 8), nullptr, + 0UL) != 0) + s.mode = -1; + return s; +} + +// Restore a previously-captured memory policy exactly. No-op if the capture +// failed. Warns (does not throw) if the OS declines. +inline void restore_thread_mempolicy(const mempolicy_state &s) { + if (s.mode < 0) + return; + if (syscall(SYS_set_mempolicy, s.mode, s.nodemask, + static_cast(sizeof(s.nodemask) * 8)) != 0) + affinity_warn("failed to restore prior thread mempolicy: " + + std::string(std::strerror(errno)) + + "; thread may remain bound to its temporary policy"); +} + // Pin the CALLING thread's CPU affinity to an explicit list of core ids. // No restore. cores.empty() = no-op. Throws std::invalid_argument if any core // id is out of range. Warns (does not throw) if the OS declines an otherwise @@ -190,12 +226,7 @@ inline std::vector current_thread_cpuset() { #else // non-Linux: no-ops -enum class mempolicy_mode { preferred, bind }; -inline void -bind_this_thread_to_numa_node(int node, - mempolicy_mode = mempolicy_mode::preferred) { - if (node < 0) - return; +inline void warn_numa_unsupported_once() { static bool warned = false; if (!warned) { affinity_warn( @@ -203,13 +234,26 @@ bind_this_thread_to_numa_node(int node, warned = true; } } -inline void -bind_region_to_numa_node(void *, std::size_t, int, - mempolicy_mode = mempolicy_mode::preferred) {} + +inline void bind_this_thread_to_numa_node( + int node, + cudaq::qec::mempolicy_mode = cudaq::qec::mempolicy_mode::preferred, + bool /*apply_mempolicy*/ = true) { + if (node < 0) + return; + warn_numa_unsupported_once(); +} +inline void bind_region_to_numa_node( + void *, std::size_t, int, + cudaq::qec::mempolicy_mode = cudaq::qec::mempolicy_mode::preferred) {} inline int current_thread_mempolicy_mode() { return -1; } inline void set_thread_cpu_affinity(const std::vector &) {} inline std::vector current_thread_cpuset() { return {}; } +struct mempolicy_state {}; +inline mempolicy_state capture_thread_mempolicy() { return {}; } +inline void restore_thread_mempolicy(const mempolicy_state &) {} + #endif } // namespace cudaq::qec::detail_affinity diff --git a/libs/qec/lib/realtime/qec_realtime_session.cpp b/libs/qec/lib/realtime/qec_realtime_session.cpp index 17de5789a..1bc1535d2 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.cpp +++ b/libs/qec/lib/realtime/qec_realtime_session.cpp @@ -337,6 +337,8 @@ void qec_realtime_session::initialize() { int chosen = -1; bool conflict = false; for (const auto &d : decoders_) { + if (!d) + continue; const int n = d->numa_node_id(); if (n < 0) continue; @@ -889,7 +891,13 @@ void qec_realtime_session::start_host_loop() { const int node = session_numa_node_; host_loop_thread_ = std::thread([this, node]() { // Persistent bind: this thread does nothing but decode for its lifetime. - cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(node); + try { + cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(node); + } catch (const std::exception &e) { + cudaq::qec::detail_affinity::affinity_warn( + "host loop thread NUMA bind failed: " + std::string(e.what()) + + " — continuing without affinity"); + } cudaq_host_dispatcher_loop(&host_ctx_); }); return; @@ -998,8 +1006,22 @@ void qec_realtime_session::start_host_loop() { host_ctx_.io_ctxs_dev = io_ctxs_dev_; host_ctx_.skip_stream_sweep = false; - host_loop_thread_ = - std::thread([this]() { cudaq_host_dispatcher_loop(&host_ctx_); }); + { + const int node = session_numa_node_; + host_loop_thread_ = std::thread([this, node]() { + // Mirror the HOST-mode dispatch thread's pinning above: the CPU-side + // monitor thread benefits from the same NUMA locality regardless of + // dispatch mode. + try { + cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(node); + } catch (const std::exception &e) { + cudaq::qec::detail_affinity::affinity_warn( + "host loop thread NUMA bind failed: " + std::string(e.what()) + + " — continuing without affinity"); + } + cudaq_host_dispatcher_loop(&host_ctx_); + }); + } } //============================================================================== diff --git a/libs/qec/unittests/CMakeLists.txt b/libs/qec/unittests/CMakeLists.txt index 28d115012..4af6462b9 100644 --- a/libs/qec/unittests/CMakeLists.txt +++ b/libs/qec/unittests/CMakeLists.txt @@ -40,11 +40,6 @@ target_include_directories(test_decoders PRIVATE ${CUDAToolkit_INCLUDE_DIRS}) add_dependencies(CUDAQXQECUnitTests test_decoders) gtest_discover_tests(test_decoders) -add_executable(test_decoder_pool test_decoder_pool.cpp decoders/sample_decoder.cpp) -target_link_libraries(test_decoder_pool PRIVATE GTest::gtest_main cudaq-qec cudaq::cudaq libstim CUDA::cudart) -target_include_directories(test_decoder_pool PRIVATE ${CUDAToolkit_INCLUDE_DIRS}) -add_dependencies(CUDAQXQECUnitTests test_decoder_pool) -gtest_discover_tests(test_decoder_pool) add_executable(test_decoders_yaml test_decoders_yaml.cpp decoders/sample_decoder.cpp) target_link_libraries(test_decoders_yaml PRIVATE @@ -181,7 +176,11 @@ if(CUDAQ_QEC_TRT_DECODER_ENABLED AND CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(A target_compile_definitions(test_trt_decoder PRIVATE TRT_TEST_ONNX_PATH="${CMAKE_CURRENT_SOURCE_DIR}/../../../assets/tests/surface_code_decoder.onnx") add_dependencies(CUDAQXQECUnitTests test_trt_decoder) - gtest_discover_tests(test_trt_decoder) + # Preload the affinity syscall shim so the guard tests can assert syscall + # counts (they degrade to placement-only asserts without it). + add_dependencies(test_trt_decoder affinity_syscall_shim) + gtest_discover_tests(test_trt_decoder + PROPERTIES ENVIRONMENT "LD_PRELOAD=$") endif() # ============================================================================== diff --git a/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp b/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp index c345fc95a..34bfd250d 100644 --- a/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp +++ b/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp @@ -162,6 +162,24 @@ TEST(PyMatchingRealtime, RejectsOversizedSyndromeRequest) { session.finalize(); } +TEST(PyMatchingRealtime, InitializeToleratesSparseDecoderIds) { + auto decoders = make_pymatching_decoders(/*h_vec=*/{1, 0, 1, 1, 0, 1}, + /*syndrome_size=*/3, + /*block_size=*/2); + auto other = make_pymatching_decoders(/*h_vec=*/{1, 0, 1, 1, 0, 1}, + /*syndrome_size=*/3, + /*block_size=*/2); + other[0]->set_decoder_id(2); + // Gap at index 1: decoder ids {0, 2}, mirroring a production config where + // decoder_config ids are not contiguous. + decoders.push_back(nullptr); + decoders.push_back(std::move(other[0])); + + cudaq::qec::realtime::qec_realtime_session session(decoders); + session.initialize(); + session.finalize(); +} + TEST(PyMatchingRealtime, ConfiguresViaRealtimeDecoderConfig) { namespace config = cudaq::qec::decoding::config; diff --git a/libs/qec/unittests/decoders/sample_decoder.cpp b/libs/qec/unittests/decoders/sample_decoder.cpp index 9bc9efae3..f09d71cde 100644 --- a/libs/qec/unittests/decoders/sample_decoder.cpp +++ b/libs/qec/unittests/decoders/sample_decoder.cpp @@ -10,6 +10,11 @@ #include #include +#if defined(__linux__) +#include +#include +#endif + using namespace cudaqx; namespace cudaq::qec { @@ -51,88 +56,68 @@ class sample_decoder : public decoder { CUDAQ_EXT_PT_REGISTER_TYPE(sample_decoder) -/// @brief Test-only decoder shaped like a typical GPU decoder: it overrides -/// ONLY the single-syndrome decode() (NOT decode_batch) and "allocates lazily" -/// on first decode(). It reports, via result.result[0], the CUDA device its -/// allocation actually landed on — so a test can assert cuda_device_id was -/// honored on whichever entry point invoked it. -class device_probe_decoder : public decoder { -public: - device_probe_decoder(const cudaq::qec::sparse_binary_matrix &H, - const cudaqx::heterogeneous_map ¶ms) - : decoder(H) {} +/// @brief Test-only decoder that records the calling thread's placement -- +/// the CUDA device a real allocation lands on and the raw MPOL_* mempolicy +/// mode -- both DURING construction (inside decoder::get()'s guarded window) +/// and inside each decode() call, and echoes them through result.result: +/// [0] = CUDA device at construction [1] = CUDA device inside decode() +/// [2] = mempolicy mode at construction [3] = mempolicy mode inside decode() +/// Unavailable probes (no CUDA / non-Linux / blocked syscall) report -1. +class placement_probe_decoder : public decoder { +private: + int ctor_device_ = -1; + int ctor_mempolicy_ = -1; - virtual decoder_result decode(const std::vector &syndrome) override { - decoder_result result; - result.converged = true; + static int current_cuda_device() { int dev = -1; void *p = nullptr; if (cudaMalloc(&p, 16) == cudaSuccess && p) { cudaPointerAttributes attr{}; if (cudaPointerGetAttributes(&attr, p) == cudaSuccess) - dev = attr.device; // device the lazy allocation landed on + dev = attr.device; // device the allocation actually landed on cudaFree(p); } else { cudaGetDevice(&dev); } - result.result = std::vector{static_cast(dev)}; - return result; + return dev; } - virtual ~device_probe_decoder() {} - - CUDAQ_EXTENSION_CUSTOM_CREATOR_FUNCTION( - device_probe_decoder, static std::unique_ptr create( - const cudaq::qec::decoder_init &init, - const cudaqx::heterogeneous_map ¶ms) { - return cudaq::qec::make_pcm_decoder(init, params); - }) -}; - -CUDAQ_EXT_PT_REGISTER_TYPE(device_probe_decoder) - -/// @brief Test-only decoder shaped like a typical GPU decoder that allocates -/// eagerly at construction (like production GPU decoders), so a test can -/// assert the construct-time device guard placed it correctly. The device its -/// constructor-time allocation landed on is recorded in a member and echoed -/// back by decode() via result.result[0]. -class eager_device_probe_decoder : public decoder { -private: - int device_ = -1; + static int current_mempolicy_mode() { + int mode = -1; +#if defined(__linux__) + syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL); +#endif + return mode; + } public: - eager_device_probe_decoder(const cudaq::qec::sparse_binary_matrix &H, - const cudaqx::heterogeneous_map ¶ms) - : decoder(H) { - void *p = nullptr; - if (cudaMalloc(&p, 16) == cudaSuccess && p) { - cudaPointerAttributes attr{}; - if (cudaPointerGetAttributes(&attr, p) == cudaSuccess) - device_ = attr.device; // device the eager allocation landed on - cudaFree(p); - } else { - cudaGetDevice(&device_); - } - } + placement_probe_decoder(const cudaq::qec::sparse_binary_matrix &H, + const cudaqx::heterogeneous_map ¶ms) + : decoder(H), ctor_device_(current_cuda_device()), + ctor_mempolicy_(current_mempolicy_mode()) {} virtual decoder_result decode(const std::vector &syndrome) override { decoder_result result; result.converged = true; - result.result = std::vector{static_cast(device_)}; + result.result = + std::vector{static_cast(ctor_device_), + static_cast(current_cuda_device()), + static_cast(ctor_mempolicy_), + static_cast(current_mempolicy_mode())}; return result; } - virtual ~eager_device_probe_decoder() {} + virtual ~placement_probe_decoder() {} CUDAQ_EXTENSION_CUSTOM_CREATOR_FUNCTION( - eager_device_probe_decoder, static std::unique_ptr create( - const cudaq::qec::decoder_init &init, - const cudaqx::heterogeneous_map ¶ms) { - return cudaq::qec::make_pcm_decoder(init, - params); + placement_probe_decoder, static std::unique_ptr create( + const cudaq::qec::decoder_init &init, + const cudaqx::heterogeneous_map ¶ms) { + return cudaq::qec::make_pcm_decoder(init, + params); }) }; -CUDAQ_EXT_PT_REGISTER_TYPE(eager_device_probe_decoder) +CUDAQ_EXT_PT_REGISTER_TYPE(placement_probe_decoder) } // namespace cudaq::qec diff --git a/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp b/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp index 96c840afa..eca2e4cc0 100644 --- a/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp +++ b/libs/qec/unittests/decoders/trt_decoder/test_trt_decoder.cpp @@ -8,21 +8,34 @@ #include "trt_test_data.h" #include "cudaq/qec/decoder.h" -#include "cudaq/qec/decoder_pool.h" #include "cudaq/qec/trt_decoder_internal.h" #include #include +#include #include #include #include #include #include +#include #include #include +#if defined(__linux__) +#include +#include +#include +#include +#endif + using namespace cudaq::qec; +// Resolved from the LD_PRELOAD affinity shim when preloaded; weak so the +// binary still links (and counter asserts are skipped) without it. +extern "C" __attribute__((weak)) void cudaqx_affinity_syscall_reset(); +extern "C" __attribute__((weak)) long cudaqx_set_mempolicy_count(); + static bool gpu_available() { int count = 0; return cudaGetDeviceCount(&count) == cudaSuccess && count > 0; @@ -841,13 +854,9 @@ TEST_F(TRTDecoderTest, CudaDeviceId_DecodeAsyncOnGpu1) { << trt_output << ", expected " << expected_output; } -// Composition check: two real trt decoders, each configured for a separate GPU, -// run concurrently through the pool and both converge to the correct output. -// This exercises a heavy production decoder end-to-end (not a probe). The -// precise per-worker device placement is asserted by the eager-probe pool test; -// here the value is that a real decoder composes with the pool and stays -// correct. -TEST_F(TRTDecoderTest, PoolRunsTwoTrtDecodersConcurrently) { +// Two real trt decoders, each pinned to a separate GPU, run concurrently from +// two independent threads and both converge to the correct output. +TEST_F(TRTDecoderTest, TwoTrtDecodersConcurrently) { if (!gpu_available()) GTEST_SKIP() << "No CUDA GPU available"; int count = 0; @@ -858,8 +867,6 @@ TEST_F(TRTDecoderTest, PoolRunsTwoTrtDecodersConcurrently) { if (!std::filesystem::exists(onnx_path)) GTEST_SKIP() << "ONNX model not found: " << onnx_path; - cudaSetDevice(0); // calling thread stays on the default device - std::size_t num_detectors = NUM_DETECTORS; cudaqx::tensor H_mat({num_detectors, num_detectors}); for (std::size_t i = 0; i < num_detectors; ++i) @@ -872,57 +879,330 @@ TEST_F(TRTDecoderTest, PoolRunsTwoTrtDecodersConcurrently) { params1.insert("onnx_load_path", onnx_path); params1.insert("cuda_device_id", 1); - std::vector specs; - specs.push_back({0, "trt_decoder", H_mat, params0}); - specs.push_back({1, "trt_decoder", H_mat, params1}); - - // Informational only: engine memory landing on each assigned GPU is - // reported for a human to see, but deltas are noisy so we don't assert. - auto free_bytes = [](int device) -> std::size_t { - int prev = 0; - cudaGetDevice(&prev); - cudaSetDevice(device); - std::size_t free = 0, total = 0; - cudaMemGetInfo(&free, &total); - cudaSetDevice(prev); - return free; - }; - std::size_t free0_before = free_bytes(0); - std::size_t free1_before = free_bytes(1); - - std::unique_ptr pool; + std::unique_ptr dec0, dec1; try { - pool = std::make_unique(std::move(specs)); + dec0 = decoder::get("trt_decoder", H_mat, params0); + dec1 = decoder::get("trt_decoder", H_mat, params1); } catch (const std::exception &e) { GTEST_SKIP() << "TRT construction failed: " << e.what(); } - std::size_t free0_after = free_bytes(0); - std::size_t free1_after = free_bytes(1); - std::cout << "GPU 0 free-memory drop: " - << (free0_before - free0_after) / (1024 * 1024) << " MiB\n"; - std::cout << "GPU 1 free-memory drop: " - << (free1_before - free1_after) / (1024 * 1024) << " MiB\n"; - std::vector syndrome(TEST_INPUTS[0].begin(), TEST_INPUTS[0].end()); - std::unordered_map>> work; - work[0] = {syndrome}; - work[1] = {syndrome}; - auto res = pool->decode_all(work); - - ASSERT_FALSE(res[0].empty()); - ASSERT_FALSE(res[1].empty()); - EXPECT_TRUE(res[0][0].converged); - EXPECT_TRUE(res[1][0].converged); - ASSERT_FALSE(res[0][0].result.empty()); - ASSERT_FALSE(res[1][0].result.empty()); + + cudaq::qec::decoder_result res0, res1; + std::exception_ptr ex0, ex1; + std::thread t0([&] { + try { + res0 = dec0->decode(syndrome); + } catch (...) { + ex0 = std::current_exception(); + } + }); + std::thread t1([&] { + try { + res1 = dec1->decode(syndrome); + } catch (...) { + ex1 = std::current_exception(); + } + }); + t0.join(); + t1.join(); + + if (ex0) + std::rethrow_exception(ex0); + if (ex1) + std::rethrow_exception(ex1); float expected_output = TEST_OUTPUTS[0][0]; - EXPECT_LT(std::abs(res[0][0].result[0] - expected_output), 1e-4f) - << "GPU-0 pool decode differs from expected: got " << res[0][0].result[0] + EXPECT_TRUE(res0.converged); + ASSERT_FALSE(res0.result.empty()); + EXPECT_LT(std::abs(res0.result[0] - expected_output), 1e-4f) + << "GPU-0 decode differs from expected: got " << res0.result[0] << ", expected " << expected_output; - EXPECT_LT(std::abs(res[1][0].result[0] - expected_output), 1e-4f) - << "GPU-1 pool decode differs from expected: got " << res[1][0].result[0] + EXPECT_TRUE(res1.converged); + ASSERT_FALSE(res1.result.empty()); + EXPECT_LT(std::abs(res1.result[0] - expected_output), 1e-4f) + << "GPU-1 decode differs from expected: got " << res1.result[0] << ", expected " << expected_output; } + +// Container runtimes commonly block the mempolicy syscalls (seccomp without +// CAP_SYS_NICE). trt_decoder::decode_batch() degrades gracefully there (warns, +// keeps going), but a test that wants to exercise the mempolicy path must skip +// instead of fail. Mirrors the guard in test_device_affinity.cpp. +TEST(TrtDecoder, DecodeBatchAppliesNumaPolicy) { +#if defined(__linux__) + int mode = -1; + if (syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL) != 0) + GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; +#endif + if (!gpu_available()) + GTEST_SKIP() << "No CUDA GPU available"; + std::string onnx_path = get_onnx_asset_path(); + if (!std::filesystem::exists(onnx_path)) + GTEST_SKIP() << "ONNX model not found: " << onnx_path; + + std::size_t num_detectors = NUM_DETECTORS; + cudaqx::tensor H({num_detectors, num_detectors}); + for (std::size_t i = 0; i < num_detectors; ++i) + H.at({i, i}) = 1; + + cudaqx::heterogeneous_map params; + params.insert("onnx_load_path", onnx_path); + params.insert("numa_node_id", 0); + + std::unique_ptr trt_decoder; + try { + trt_decoder = decoder::get("trt_decoder", H, params); + } catch (const std::exception &e) { + GTEST_SKIP() << "Failed to create TRT decoder: " << e.what(); + } + + std::vector syndrome(TEST_INPUTS[0].begin(), + TEST_INPUTS[0].end()); + std::vector results; + // decode_batch() applies the NUMA guard for the duration of the call and + // restores the thread's prior policy before returning, so this only proves + // no crash/throw occurs with a numa_node_id knob set; the mempolicy syscall + // behavior itself is covered at the primitive level by + // HardwareAffinity.MempolicyBindWhenRequested. + EXPECT_NO_THROW({ results = trt_decoder->decode_batch({syndrome}); }); + ASSERT_EQ(results.size(), 1u); + EXPECT_TRUE(results[0].converged); + ASSERT_FALSE(results[0].result.empty()); +} + +#if defined(__linux__) +// Snapshot of the calling thread's placement: CPU affinity mask plus thread +// memory-policy mode and nodemask (raw SYS_get_mempolicy, matching what the +// decode_batch guard saves/restores). +struct thread_placement { + cpu_set_t mask; + int mempolicy_mode = -1; + unsigned long nodemask[16] = {0}; +}; + +static std::optional capture_thread_placement() { + thread_placement p; + CPU_ZERO(&p.mask); + if (sched_getaffinity(0, sizeof(p.mask), &p.mask) != 0) + return std::nullopt; + if (syscall(SYS_get_mempolicy, &p.mempolicy_mode, p.nodemask, + sizeof(p.nodemask) * 8, nullptr, 0UL) != 0) + return std::nullopt; + return p; +} + +static void expect_same_placement(const thread_placement &before, + const thread_placement &after) { + EXPECT_TRUE(CPU_EQUAL(&before.mask, &after.mask)) + << "CPU affinity mask changed across decode_batch()"; + EXPECT_EQ(before.mempolicy_mode, after.mempolicy_mode) + << "thread mempolicy mode changed across decode_batch()"; + EXPECT_EQ( + 0, std::memcmp(before.nodemask, after.nodemask, sizeof(before.nodemask))) + << "thread mempolicy nodemask changed across decode_batch()"; +} + +// Shared skip guard for the placement tests below (GPU + ONNX asset + NUMA +// node 0 + mempolicy syscalls, which container seccomp commonly blocks). +static std::optional placement_test_skip_reason() { + if (!gpu_available()) + return "No CUDA GPU available"; + if (!std::filesystem::exists(get_onnx_asset_path())) + return "ONNX model not found: " + get_onnx_asset_path(); + if (!std::filesystem::exists("/sys/devices/system/node/node0")) + return "NUMA node 0 not present"; + int mode = -1; + if (syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL) != 0) + return "mempolicy syscalls unavailable (container seccomp?)"; + return std::nullopt; +} + +static void expect_first_output_correct( + const std::vector &results) { + ASSERT_EQ(results.size(), 1u); + EXPECT_TRUE(results[0].converged); + ASSERT_FALSE(results[0].result.empty()); + EXPECT_LT(std::abs(results[0].result[0] - TEST_OUTPUTS[0][0]), 1e-4f) + << "decode_batch output differs from expected: got " + << results[0].result[0] << ", expected " << TEST_OUTPUTS[0][0]; +} +#endif + +// A thread that called bind_current_thread() owns its placement: the +// decode_batch() guard must not fire on it. Placement is captured after the +// bind and must be bit-identical after decode_batch() — regression test for +// the pinning-defeat bug where the guard re-bound (and widened) an +// already-bound thread. +TEST(TrtDecoder, BoundThreadSkipsGuardInDecodeBatch) { +#if defined(__linux__) + if (auto reason = placement_test_skip_reason()) + GTEST_SKIP() << *reason; + + cudaqx::heterogeneous_map params; + params.insert("onnx_load_path", get_onnx_asset_path()); + params.insert("cuda_device_id", 0); + params.insert("numa_node_id", 0); + std::unique_ptr trt_decoder; + try { + trt_decoder = + decoder::get("trt_decoder", make_identity_h(NUM_DETECTORS), params); + } catch (const std::exception &e) { + GTEST_SKIP() << "Failed to create TRT decoder: " << e.what(); + } + + std::vector syndrome(TEST_INPUTS[0].begin(), + TEST_INPUTS[0].end()); + // bind_current_thread() pins persistently (no restore), so run on a worker + // thread to keep the gtest main thread's placement intact for later tests. + std::optional before, after; + std::vector results; + std::exception_ptr err; + // A guard that fires and restores perfectly is indistinguishable from a + // skipped guard by before/after placement alone; the set_mempolicy counter + // is the observable difference (TRT itself never calls set_mempolicy). + const bool have_shim = cudaqx_affinity_syscall_reset != nullptr && + cudaqx_set_mempolicy_count != nullptr; + long setmem_during_decode = 0; + std::thread worker([&] { + try { + trt_decoder->bind_current_thread(); + before = capture_thread_placement(); + if (have_shim) + cudaqx_affinity_syscall_reset(); + results = trt_decoder->decode_batch({syndrome}); + if (have_shim) + setmem_during_decode = cudaqx_set_mempolicy_count(); + after = capture_thread_placement(); + } catch (...) { + err = std::current_exception(); + } + }); + worker.join(); + if (err) + std::rethrow_exception(err); + + expect_first_output_correct(results); + ASSERT_TRUE(before.has_value()); + ASSERT_TRUE(after.has_value()); + expect_same_placement(*before, *after); + if (have_shim) + EXPECT_EQ(setmem_during_decode, 0) + << "bound decode_batch() issued set_mempolicy: the guard fired on a " + "thread that already called bind_current_thread()"; +#else + GTEST_SKIP() << "Linux-only placement test"; +#endif +} + +// Unbound caller: decode_batch() applies the NUMA guard for the duration of +// the call and must restore the caller's placement exactly before returning. +TEST(TrtDecoder, UnboundDecodeBatchRestoresPlacement) { +#if defined(__linux__) + if (auto reason = placement_test_skip_reason()) + GTEST_SKIP() << *reason; + + cudaqx::heterogeneous_map params; + params.insert("onnx_load_path", get_onnx_asset_path()); + params.insert("cuda_device_id", 0); + params.insert("numa_node_id", 0); + std::unique_ptr trt_decoder; + try { + trt_decoder = + decoder::get("trt_decoder", make_identity_h(NUM_DETECTORS), params); + } catch (const std::exception &e) { + GTEST_SKIP() << "Failed to create TRT decoder: " << e.what(); + } + + std::vector syndrome(TEST_INPUTS[0].begin(), + TEST_INPUTS[0].end()); + auto before = capture_thread_placement(); + ASSERT_TRUE(before.has_value()); + auto results = trt_decoder->decode_batch({syndrome}); + auto after = capture_thread_placement(); + ASSERT_TRUE(after.has_value()); + + expect_first_output_correct(results); + expect_same_placement(*before, *after); +#else + GTEST_SKIP() << "Linux-only placement test"; +#endif +} + +// An explicit cpu_affinity={0} bind is strictly narrower than node 0's cpuset: +// if decode_batch()'s _ScopedNuma guard fired on the bound thread it would +// widen the mask to the whole node, so "exactly CPU 0 before AND after" proves +// the explicit pin survives decode_batch(). +TEST(TrtDecoder, ExplicitCpuAffinityHonoredWhenBound) { +#if defined(__linux__) + if (auto reason = placement_test_skip_reason()) + GTEST_SKIP() << *reason; + + cudaqx::heterogeneous_map params; + params.insert("onnx_load_path", get_onnx_asset_path()); + params.insert("numa_node_id", 0); + params.insert("cpu_affinity", std::vector{0}); + std::unique_ptr trt_decoder; + try { + trt_decoder = + decoder::get("trt_decoder", make_identity_h(NUM_DETECTORS), params); + } catch (const std::exception &e) { + GTEST_SKIP() << "Failed to create TRT decoder: " << e.what(); + } + + std::vector syndrome(TEST_INPUTS[0].begin(), + TEST_INPUTS[0].end()); + cpu_set_t only_cpu0; + CPU_ZERO(&only_cpu0); + CPU_SET(0, &only_cpu0); + + std::optional before, after; + std::vector results; + std::exception_ptr err; + std::thread worker([&] { + try { + trt_decoder->bind_current_thread(); + before = capture_thread_placement(); + results = trt_decoder->decode_batch({syndrome}); + after = capture_thread_placement(); + } catch (...) { + err = std::current_exception(); + } + }); + worker.join(); + if (err) + std::rethrow_exception(err); + + expect_first_output_correct(results); + ASSERT_TRUE(before.has_value()); + ASSERT_TRUE(after.has_value()); + EXPECT_TRUE(CPU_EQUAL(&only_cpu0, &before->mask)) + << "bind_current_thread() did not honor cpu_affinity={0}"; + EXPECT_TRUE(CPU_EQUAL(&only_cpu0, &after->mask)) + << "decode_batch() widened the explicit cpu_affinity={0} pin"; + expect_same_placement(*before, *after); +#else + GTEST_SKIP() << "Linux-only placement test"; +#endif +} + +// Dependency-free regression guard: decoder::get()'s own construction-time +// CudaDeviceGuard rejects an out-of-range cuda_device_id before the plugin is +// ever instantiated, so decode_batch()'s own device-guard error-checking is +// never reached for this particular failure mode. Kept as a guard on the +// overall contract (out-of-range device ids must never silently proceed). +TEST(TrtDecoder, DecodeBatchThrowsOnOutOfRangeCudaDeviceId) { + int count = 0; + cudaGetDeviceCount(&count); + cudaqx::tensor H({2, 2}); + H.at({0, 0}) = 1; + H.at({1, 1}) = 1; + cudaqx::heterogeneous_map params; + params.insert("onnx_load_path", get_onnx_asset_path()); + params.insert("cuda_device_id", count + 5); // deliberately out of range + EXPECT_THROW( + { cudaq::qec::decoder::get("trt_decoder", H, params); }, + std::runtime_error); +} diff --git a/libs/qec/unittests/support/affinity_syscall_shim.cpp b/libs/qec/unittests/support/affinity_syscall_shim.cpp index bb5770d85..47076c4fd 100644 --- a/libs/qec/unittests/support/affinity_syscall_shim.cpp +++ b/libs/qec/unittests/support/affinity_syscall_shim.cpp @@ -1,31 +1,127 @@ -// Test-only LD_PRELOAD interposer: counts sched_setaffinity calls so a test can -// assert a bound decode loop issues none. Not linked into the library. +// Test-only LD_PRELOAD interposer: counts the placement-related syscalls so a +// test can assert a bound decode loop issues none (and an unbound one does). +// Counts four syscalls: sched_setaffinity / sched_getaffinity (glibc symbols) +// and SYS_set_mempolicy / SYS_get_mempolicy (issued through glibc's syscall(2) +// wrapper, so that wrapper is interposed too). Not linked into the library. +#ifndef _GNU_SOURCE #define _GNU_SOURCE +#endif #include +#include +#include +#include #include #include +#include +#include namespace { -std::atomic g_count{0}; +std::atomic g_setaff{0}; +std::atomic g_getaff{0}; +std::atomic g_setmem{0}; +std::atomic g_getmem{0}; + +// dlsym may itself invoke interposed functions; the thread-local flag breaks +// that recursion (the inner call sees "resolving" and reports ENOSYS instead +// of re-entering dlsym forever). +using syscall_fn = long (*)(long, long, long, long, long, long, long); +syscall_fn real_syscall_lazy() { + static std::atomic cached{nullptr}; + syscall_fn fn = cached.load(std::memory_order_acquire); + if (fn) + return fn; + static thread_local bool resolving = false; + if (resolving) + return nullptr; + resolving = true; + fn = reinterpret_cast(dlsym(RTLD_NEXT, "syscall")); + resolving = false; + if (fn) + cached.store(fn, std::memory_order_release); + return fn; } +} // namespace extern "C" { -// Interpose the glibc symbol; forward to the real one via RTLD_NEXT. +// Interpose the glibc symbols; forward to the real ones via RTLD_NEXT. int sched_setaffinity(pid_t pid, size_t cpusetsize, const cpu_set_t *mask) { static int (*real)(pid_t, size_t, const cpu_set_t *) = nullptr; if (!real) real = reinterpret_cast( dlsym(RTLD_NEXT, "sched_setaffinity")); - g_count.fetch_add(1, std::memory_order_relaxed); + g_setaff.fetch_add(1, std::memory_order_relaxed); + return real(pid, cpusetsize, mask); +} + +int sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t *mask) { + static int (*real)(pid_t, size_t, cpu_set_t *) = nullptr; + if (!real) + real = reinterpret_cast( + dlsym(RTLD_NEXT, "sched_getaffinity")); + g_getaff.fetch_add(1, std::memory_order_relaxed); return real(pid, cpusetsize, mask); } +// The mempolicy calls have no glibc wrappers; the library reaches them through +// syscall(2), so interpose that. Counts ATTEMPTS (even ones seccomp rejects). +long syscall(long number, ...) { + va_list ap; + va_start(ap, number); + long a0 = va_arg(ap, long), a1 = va_arg(ap, long), a2 = va_arg(ap, long); + long a3 = va_arg(ap, long), a4 = va_arg(ap, long), a5 = va_arg(ap, long); + va_end(ap); + switch (number) { + case SYS_set_mempolicy: + g_setmem.fetch_add(1, std::memory_order_relaxed); + break; + case SYS_get_mempolicy: + g_getmem.fetch_add(1, std::memory_order_relaxed); + // Fault injection: simulate a seccomp profile that blocks get_mempolicy + // while allowing set_mempolicy (the un-restorable-policy scenario). + // getenv per call so a test can toggle it around a single decode. + if (std::getenv("CUDAQX_SHIM_FAIL_GET_MEMPOLICY")) { + errno = EPERM; + return -1; + } + break; + case SYS_sched_setaffinity: + g_setaff.fetch_add(1, std::memory_order_relaxed); + break; + case SYS_sched_getaffinity: + g_getaff.fetch_add(1, std::memory_order_relaxed); + break; + } + syscall_fn real = real_syscall_lazy(); + if (!real) { // recursion during resolve; refuse rather than loop + errno = ENOSYS; + return -1; + } + return real(number, a0, a1, a2, a3, a4, a5); +} + // Read/reset hooks the test resolves (weakly) from the preloaded shim. +// Legacy pair kept for existing tests: count == sched_setaffinity count. long cudaqx_affinity_syscall_count() { - return g_count.load(std::memory_order_relaxed); + return g_setaff.load(std::memory_order_relaxed); } void cudaqx_affinity_syscall_reset() { - g_count.store(0, std::memory_order_relaxed); + g_setaff.store(0, std::memory_order_relaxed); + g_getaff.store(0, std::memory_order_relaxed); + g_setmem.store(0, std::memory_order_relaxed); + g_getmem.store(0, std::memory_order_relaxed); +} +// Per-syscall counters. +long cudaqx_sched_setaffinity_count() { + return g_setaff.load(std::memory_order_relaxed); +} +long cudaqx_sched_getaffinity_count() { + return g_getaff.load(std::memory_order_relaxed); +} +long cudaqx_set_mempolicy_count() { + return g_setmem.load(std::memory_order_relaxed); +} +long cudaqx_get_mempolicy_count() { + return g_getmem.load(std::memory_order_relaxed); } } diff --git a/libs/qec/unittests/support/thread_placement.h b/libs/qec/unittests/support/thread_placement.h new file mode 100644 index 000000000..0af736c05 --- /dev/null +++ b/libs/qec/unittests/support/thread_placement.h @@ -0,0 +1,79 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ +// Test-only, header-only snapshot of the calling thread's hardware placement +// (CPU affinity + NUMA mempolicy mode + optional CUDA device) so tests can +// assert a guard restored everything exactly. Included via a relative path; +// not installed. +#ifndef CUDAQ_QEC_UNITTESTS_SUPPORT_THREAD_PLACEMENT_H +#define CUDAQ_QEC_UNITTESTS_SUPPORT_THREAD_PLACEMENT_H + +#include +#include +#include +#include +#if defined(__linux__) +#include +#include +#endif + +namespace cudaq::qec::test { + +struct thread_placement { + cpu_set_t affinity{}; + int mempolicy_mode = -1; // -1: query failed / unavailable (e.g. seccomp) + int cuda_device = -1; // -1: not captured (with_cuda=false) or no CUDA + + // NOTE: issues sched_getaffinity + SYS_get_mempolicy itself; call it outside + // any shim reset/read window. cudaGetDevice only runs when with_cuda is set + // so CPU-only tests never touch the CUDA runtime. + static thread_placement capture(bool with_cuda = false) { + thread_placement p; + CPU_ZERO(&p.affinity); + (void)sched_getaffinity(0, sizeof(p.affinity), &p.affinity); +#if defined(__linux__) + if (syscall(SYS_get_mempolicy, &p.mempolicy_mode, nullptr, 0UL, nullptr, + 0UL) != 0) + p.mempolicy_mode = -1; +#endif + if (with_cuda && cudaGetDevice(&p.cuda_device) != cudaSuccess) + p.cuda_device = -1; + return p; + } + + bool operator==(const thread_placement &o) const { + return CPU_EQUAL(&affinity, &o.affinity) && + mempolicy_mode == o.mempolicy_mode && cuda_device == o.cuda_device; + } + bool operator!=(const thread_placement &o) const { return !(*this == o); } + + // Names which field differs (for assertion messages). + std::string describe_difference(const thread_placement &o) const { + std::string out; + if (!CPU_EQUAL(&affinity, &o.affinity)) + out += "cpu affinity differs (" + std::to_string(CPU_COUNT(&affinity)) + + " vs " + std::to_string(CPU_COUNT(&o.affinity)) + " cpus set); "; + if (mempolicy_mode != o.mempolicy_mode) + out += "mempolicy_mode differs (" + std::to_string(mempolicy_mode) + + " vs " + std::to_string(o.mempolicy_mode) + "); "; + if (cuda_device != o.cuda_device) + out += "cuda_device differs (" + std::to_string(cuda_device) + " vs " + + std::to_string(o.cuda_device) + "); "; + return out.empty() ? "identical" : out; + } +}; + +// gtest hook so failed EXPECT_EQ prints something readable. +inline void PrintTo(const thread_placement &p, std::ostream *os) { + *os << "{cpus_set=" << CPU_COUNT(&p.affinity) + << ", mempolicy_mode=" << p.mempolicy_mode + << ", cuda_device=" << p.cuda_device << "}"; +} + +} // namespace cudaq::qec::test + +#endif // CUDAQ_QEC_UNITTESTS_SUPPORT_THREAD_PLACEMENT_H diff --git a/libs/qec/unittests/test_decoder_pool.cpp b/libs/qec/unittests/test_decoder_pool.cpp deleted file mode 100644 index d1758e9c8..000000000 --- a/libs/qec/unittests/test_decoder_pool.cpp +++ /dev/null @@ -1,199 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ -#include "cudaq/qec/decoder_pool.h" -#include -#include - -static cudaqx::tensor makeH() { - cudaqx::tensor H({2, 3}); - return H; -} - -// Orchestration (no GPU needed): each id's syndromes are routed to its own -// decoder and aggregated back under that id. -TEST(DecoderPool, RoutesAndAggregatesPerId) { - using cudaq::qec::float_t; - std::vector specs; - specs.push_back({7, "multi_error_lut", makeH(), {}}); - specs.push_back({9, "multi_error_lut", makeH(), {}}); - cudaq::qec::decoder_pool pool(std::move(specs)); - - std::unordered_map>> work; - work[7] = {{0.1f, 0.1f}, {0.2f, 0.2f}}; - work[9] = {{0.3f, 0.3f}}; - auto res = pool.decode_all(work); - ASSERT_EQ(res.size(), 2u); - EXPECT_EQ(res[7].size(), 2u); - EXPECT_EQ(res[9].size(), 1u); -} - -TEST(DecoderPool, UnknownIdThrows) { - using cudaq::qec::float_t; - std::vector specs; - specs.push_back({1, "multi_error_lut", makeH(), {}}); - cudaq::qec::decoder_pool pool(std::move(specs)); - std::unordered_map>> work; - work[42] = {{0.1f, 0.1f}}; - EXPECT_THROW(pool.decode_all(work), std::runtime_error); -} - -// Placement: each worker decodes on its assigned GPU. Uses the probe decoder, -// which reports (via result.result[0]) the device its allocation landed on. -TEST(DecoderPool, EachWorkerRunsOnAssignedGpu) { - using cudaq::qec::float_t; - int n = 0; - cudaGetDeviceCount(&n); - if (n < 2) - GTEST_SKIP() << "needs >= 2 GPUs"; - cudaSetDevice(0); - std::vector specs; - cudaqx::heterogeneous_map o0; - o0.insert("cuda_device_id", 0); - cudaqx::heterogeneous_map o1; - o1.insert("cuda_device_id", 1); - specs.push_back({0, "device_probe_decoder", makeH(), o0}); - specs.push_back({1, "device_probe_decoder", makeH(), o1}); - cudaq::qec::decoder_pool pool(std::move(specs)); - std::unordered_map>> work; - work[0] = {{0.1f, 0.1f}}; - work[1] = {{0.1f, 0.1f}}; - auto res = pool.decode_all(work); - ASSERT_EQ(res[0].size(), 1u); - ASSERT_EQ(res[1].size(), 1u); - EXPECT_EQ(static_cast(res[0][0].result[0]), 0); - EXPECT_EQ(static_cast(res[1][0].result[0]), 1); -} - -// Validates the real closed nv-qldpc decoder, when compiled against this -// base, running two decoders concurrently on distinct GPUs through the pool. -// Skips wherever nv-qldpc-decoder isn't installed (e.g. this dev build). -TEST(DecoderPool, NvQldpcRunsOnDistinctGpusWhenAvailable) { - using cudaq::qec::float_t; - std::vector H_vec = {1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, - 1, 0, 1, 0, 0, 1, 0, 1, 1, 1}; - cudaqx::tensor H; - H.copy(H_vec.data(), {3, 7}); - try { - auto d = cudaq::qec::decoder::get("nv-qldpc-decoder", H); - } catch (const std::exception &) { - GTEST_SKIP() << "nv-qldpc-decoder not available"; - } - - int n = 0; - cudaGetDeviceCount(&n); - if (n < 2) - GTEST_SKIP() << "needs >= 2 GPUs"; - - std::vector specs; - // nv-qldpc's GPU batched-decode path requires sparse mode. - cudaqx::heterogeneous_map o0; - o0.insert("cuda_device_id", 0); - o0.insert("use_sparsity", true); - cudaqx::heterogeneous_map o1; - o1.insert("cuda_device_id", 1); - o1.insert("use_sparsity", true); - specs.push_back({0, "nv-qldpc-decoder", H, o0}); - specs.push_back({1, "nv-qldpc-decoder", H, o1}); - cudaq::qec::decoder_pool pool(std::move(specs)); - - std::unordered_map>> work; - work[0] = {{1.0f, 0.0f, 1.0f}}; - work[1] = {{1.0f, 0.0f, 1.0f}}; - auto res = pool.decode_all(work); - ASSERT_EQ(res[0].size(), 1u); - ASSERT_EQ(res[1].size(), 1u); - EXPECT_TRUE(res[0][0].converged); - EXPECT_TRUE(res[1][0].converged); -} - -// Streaming: submit returns immediately without waiting on any decode, so a -// caller can keep several chunks in flight per id at once and drain the -// futures as they resolve. -TEST(DecoderPool, StreamsChunksPerIdConcurrently) { - using cudaq::qec::float_t; - std::vector specs; - specs.push_back({7, "multi_error_lut", makeH(), {}}); - specs.push_back({9, "multi_error_lut", makeH(), {}}); - cudaq::qec::decoder_pool pool(std::move(specs)); - - std::vector< - std::pair>>> - in_flight; - for (int round = 0; round < 4; ++round) { - in_flight.emplace_back(7, pool.submit(7, {{0.1f, 0.1f}, {0.2f, 0.2f}})); - in_flight.emplace_back(9, pool.submit(9, {{0.3f, 0.3f}})); - } - - std::unordered_map counts; - for (auto &[id, fut] : in_flight) - counts[id] += fut.get().size(); - EXPECT_EQ(counts[7], 8u); - EXPECT_EQ(counts[9], 4u); -} - -// Placement across streamed chunks: the worker binds its GPU once at -// construction, so every chunk it decodes over time (not just the first) -// must land on that same device. -TEST(DecoderPool, StreamingKeepsWorkerPinnedAcrossChunks) { - using cudaq::qec::float_t; - int n = 0; - cudaGetDeviceCount(&n); - if (n < 2) - GTEST_SKIP() << "needs >= 2 GPUs"; - cudaSetDevice(0); - std::vector specs; - cudaqx::heterogeneous_map o0; - o0.insert("cuda_device_id", 0); - cudaqx::heterogeneous_map o1; - o1.insert("cuda_device_id", 1); - specs.push_back({0, "device_probe_decoder", makeH(), o0}); - specs.push_back({1, "device_probe_decoder", makeH(), o1}); - cudaq::qec::decoder_pool pool(std::move(specs)); - - std::vector< - std::pair>>> - in_flight; - for (int round = 0; round < 4; ++round) { - in_flight.emplace_back(0, pool.submit(0, {{0.1f, 0.1f}})); - in_flight.emplace_back(1, pool.submit(1, {{0.1f, 0.1f}})); - } - - for (auto &[id, fut] : in_flight) { - auto r = fut.get(); - ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(static_cast(r[0].result[0]), id); - } -} - -// Placement at construction time: each worker's decoder allocates GPU memory -// in its constructor (like production GPU decoders), and the construct-time -// device guard must already have it on the right device before any decode. -TEST(DecoderPool, EachWorkerConstructsOnAssignedGpu) { - using cudaq::qec::float_t; - int n = 0; - cudaGetDeviceCount(&n); - if (n < 2) - GTEST_SKIP() << "needs >= 2 GPUs"; - cudaSetDevice(0); - std::vector specs; - cudaqx::heterogeneous_map o0; - o0.insert("cuda_device_id", 0); - cudaqx::heterogeneous_map o1; - o1.insert("cuda_device_id", 1); - specs.push_back({0, "eager_device_probe_decoder", makeH(), o0}); - specs.push_back({1, "eager_device_probe_decoder", makeH(), o1}); - cudaq::qec::decoder_pool pool(std::move(specs)); - std::unordered_map>> work; - work[0] = {{0.1f, 0.1f}}; - work[1] = {{0.1f, 0.1f}}; - auto res = pool.decode_all(work); - ASSERT_EQ(res[0].size(), 1u); - ASSERT_EQ(res[1].size(), 1u); - EXPECT_EQ(static_cast(res[0][0].result[0]), 0); - EXPECT_EQ(static_cast(res[1][0].result[0]), 1); -} diff --git a/libs/qec/unittests/test_decoders.cpp b/libs/qec/unittests/test_decoders.cpp index 462b95685..0e8438aea 100644 --- a/libs/qec/unittests/test_decoders.cpp +++ b/libs/qec/unittests/test_decoders.cpp @@ -19,7 +19,9 @@ #include #include #if defined(__linux__) +#include #include +#include #endif namespace { @@ -1343,18 +1345,18 @@ TEST(HardwarePinningGpu, RawDecodeOnPinnedThreadUsesAssignedGpu) { cudaqx::tensor H({2, 3}); cudaqx::heterogeneous_map p; p.insert("cuda_device_id", 1); - auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); + auto d = cudaq::qec::decoder::get("placement_probe_decoder", H, p); int observed = -1; std::size_t sz = 0; std::thread worker([&] { d->bind_current_thread(); // owns this decoder; pins current device to 1 auto r = d->decode(std::vector{0.1f, 0.1f}); sz = r.result.size(); - if (sz) - observed = static_cast(r.result[0]); + if (sz > 1) + observed = static_cast(r.result[1]); // device inside decode() }); worker.join(); - ASSERT_EQ(sz, 1u); + ASSERT_EQ(sz, 4u); EXPECT_EQ(observed, 1) << "decode() on a pinned worker did not use the assigned GPU"; } @@ -1372,20 +1374,20 @@ TEST(HardwarePinningGpu, MultiPatchOnPinnedThreadsUseDistinctGpus) { p0.insert("cuda_device_id", 0); cudaqx::heterogeneous_map p1; p1.insert("cuda_device_id", 1); - auto d0 = cudaq::qec::decoder::get("device_probe_decoder", H, p0); - auto d1 = cudaq::qec::decoder::get("device_probe_decoder", H, p1); + auto d0 = cudaq::qec::decoder::get("placement_probe_decoder", H, p0); + auto d1 = cudaq::qec::decoder::get("placement_probe_decoder", H, p1); int o0 = -1, o1 = -1; std::thread t0([&] { d0->bind_current_thread(); auto r = d0->decode(std::vector{0.1f, 0.1f}); - if (r.result.size()) - o0 = static_cast(r.result[0]); + if (r.result.size() > 1) + o0 = static_cast(r.result[1]); // device inside decode() }); std::thread t1([&] { d1->bind_current_thread(); auto r = d1->decode(std::vector{0.1f, 0.1f}); - if (r.result.size()) - o1 = static_cast(r.result[0]); + if (r.result.size() > 1) + o1 = static_cast(r.result[1]); // device inside decode() }); t0.join(); t1.join(); @@ -1404,10 +1406,11 @@ TEST(HardwarePinningGpu, DecodeAsyncHonorsCudaDeviceId) { cudaqx::tensor H({2, 3}); cudaqx::heterogeneous_map p; p.insert("cuda_device_id", 1); - auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); + auto d = cudaq::qec::decoder::get("placement_probe_decoder", H, p); auto fut = d->decode_async(std::vector{0.1f, 0.1f}); auto r = fut.get(); - EXPECT_EQ(static_cast(r.result[0]), 1) + ASSERT_EQ(r.result.size(), 4u); + EXPECT_EQ(static_cast(r.result[1]), 1) << "decode_async did not honor cuda_device_id"; } @@ -1421,43 +1424,31 @@ TEST(HardwarePinningGpu, DecodeBatchHonorsCudaDeviceId) { cudaqx::tensor H({2, 3}); cudaqx::heterogeneous_map p; p.insert("cuda_device_id", 1); - auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); + auto d = cudaq::qec::decoder::get("placement_probe_decoder", H, p); auto rs = d->decode_batch( std::vector>{{0.1f, 0.1f}}); ASSERT_EQ(rs.size(), 1u); - EXPECT_EQ(static_cast(rs[0].result[0]), 1) + ASSERT_EQ(rs[0].result.size(), 4u); + EXPECT_EQ(static_cast(rs[0].result[1]), 1) << "decode_batch did not honor cuda_device_id"; } -// The base-controlled decode(tensor) overload and enqueue_syndrome() both -// check for a mismatch between an explicit cuda_device_id and the caller's -// current CUDA device, warning once on stderr rather than silently decoding -// on the wrong GPU. -TEST(HardwarePinningGpu, WarnsOnceWhenDecodingOnMismatchedDevice) { +TEST(DecoderAffinity, DecodeTensorAppliesGuardWithoutBindCurrentThread) { int n = 0; cudaGetDeviceCount(&n); if (n < 2) GTEST_SKIP() << "needs >= 2 GPUs"; - cudaSetDevice(0); // current device 0, but decoder wants 1 -> mismatch + cudaSetDevice(0); cudaqx::tensor H({2, 3}); - cudaqx::heterogeneous_map p; - p.insert("cuda_device_id", 1); - auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); - cudaqx::tensor syn; - std::vector sv = {1, 0}; - syn.copy(sv.data(), {2}); - testing::internal::CaptureStderr(); - (void)d->decode(syn); // base tensor overload -> warn_if_device_mismatch - (void)d->decode(syn); // second call must NOT warn again (once) - std::string err = testing::internal::GetCapturedStderr(); - EXPECT_NE(err.find("cuda_device_id 1"), std::string::npos) - << "expected a device-mismatch warning on stderr"; - // warn-once: the phrase appears exactly once - auto first = err.find("but decoding on current device"); - ASSERT_NE(first, std::string::npos); - EXPECT_EQ(err.find("but decoding on current device", first + 1), - std::string::npos) - << "device-mismatch warning should fire at most once"; + cudaqx::heterogeneous_map o; + o.insert("cuda_device_id", 1); + auto d = cudaq::qec::decoder::get("placement_probe_decoder", H, o); + cudaqx::tensor syndrome({2}); + auto result = d->decode(syndrome); + ASSERT_EQ(result.result.size(), 4u); + EXPECT_EQ(static_cast(result.result[1]), 1) + << "decode(tensor) must land on cuda_device_id even without an " + "explicit bind_current_thread() call"; } // decode_on_pinned_thread spawns a worker, binds it to the decoder's assigned @@ -1471,11 +1462,11 @@ TEST(HardwarePinningGpu, DecodeOnPinnedThreadUsesAssignedGpu) { cudaqx::tensor H({2, 3}); cudaqx::heterogeneous_map p; p.insert("cuda_device_id", 1); - auto d = cudaq::qec::decoder::get("device_probe_decoder", H, p); + auto d = cudaq::qec::decoder::get("placement_probe_decoder", H, p); auto r = d->decode_on_pinned_thread(std::vector{0.1f, 0.1f}); - ASSERT_EQ(r.result.size(), 1u); - EXPECT_EQ(static_cast(r.result[0]), 1) + ASSERT_EQ(r.result.size(), 4u); + EXPECT_EQ(static_cast(r.result[1]), 1) << "decode_on_pinned_thread did not run on the assigned GPU"; } @@ -1505,3 +1496,168 @@ TEST(HardwarePinning, NumaAutoDeriveUnknownIsInfoNotThrow) { EXPECT_NO_THROW( { EXPECT_EQ(cudaq::qec::numa_node_for_cuda_device(-1), -1); }); } + +TEST(DecoderAffinity, ConstructionTimeGuardHonorsExplicitBindMempolicy) { +#if defined(__linux__) + int probe = -1; + if (syscall(SYS_get_mempolicy, &probe, nullptr, 0UL, nullptr, 0UL) != 0) + GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map o; + o.insert("numa_node_id", 0); + o.insert("mempolicy", std::string("bind")); + auto d = cudaq::qec::decoder::get("placement_probe_decoder", H, o); + auto result = d->decode(std::vector{0.0, 0.0}); + ASSERT_EQ(result.result.size(), 4u); + EXPECT_EQ(static_cast(result.result[2]), MPOL_BIND) + << "decoder::get()'s construction-time guard must honor an explicit " + "mempolicy:\"bind\" knob, not default to preferred"; +#else + GTEST_SKIP() << "Linux-only"; +#endif +} + +// decode(tensor) is a guarded entry point, but the guard must be SKIPPED on +// the exact thread that called bind_current_thread(): after the bind, the +// thread's mempolicy is deliberately reset to a sentinel (MPOL_DEFAULT); a +// skipped guard leaves the sentinel visible inside decode() and untouched +// after it, while a wrongly applied guard would overwrite it with MPOL_BIND. +TEST(DecoderAffinity, DecodeTensorSkipsGuardOnBoundThread) { +#if defined(__linux__) + int probe = -1; + if (syscall(SYS_get_mempolicy, &probe, nullptr, 0UL, nullptr, 0UL) != 0) + GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map o; + o.insert("numa_node_id", 0); + o.insert("mempolicy", std::string("bind")); + auto d = cudaq::qec::decoder::get("placement_probe_decoder", H, o); + int mode_after_bind = -1, mode_in_decode = -1, mode_after_decode = -1; + std::size_t sz = 0; + std::thread worker([&] { // worker so the bind never leaks into other tests + d->bind_current_thread(); + syscall(SYS_get_mempolicy, &mode_after_bind, nullptr, 0UL, nullptr, 0UL); + // Sentinel distinct from the decoder's "bind" knob. + syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); + cudaqx::tensor syndrome({2}); + auto r = d->decode(syndrome); + sz = r.result.size(); + if (sz == 4) + mode_in_decode = static_cast(r.result[3]); + syscall(SYS_get_mempolicy, &mode_after_decode, nullptr, 0UL, nullptr, 0UL); + }); + worker.join(); + EXPECT_EQ(mode_after_bind, MPOL_BIND) + << "bind_current_thread did not establish the requested mempolicy"; + ASSERT_EQ(sz, 4u); + EXPECT_EQ(mode_in_decode, MPOL_DEFAULT) + << "decode(tensor) re-applied the guard on the bound thread"; + EXPECT_EQ(mode_after_decode, MPOL_DEFAULT) + << "decode(tensor) changed the bound thread's placement"; +#else + GTEST_SKIP() << "Linux-only"; +#endif +} + +// A failed bind_current_thread (valid numa_node_id, out-of-range cpu_affinity +// core id) must throw std::invalid_argument and roll back every side effect: +// mempolicy mode and CPU affinity mask are restored, and the decoder is NOT +// left bound -- a subsequent guarded decode still applies the guard. +TEST(DecoderAffinity, FailedBindThrowsAndFullyRestoresThread) { +#if defined(__linux__) + int probe = -1; + if (syscall(SYS_get_mempolicy, &probe, nullptr, 0UL, nullptr, 0UL) != 0) + GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map o; + o.insert("numa_node_id", 0); + o.insert("mempolicy", std::string("bind")); + o.insert("cpu_affinity", std::vector{CPU_SETSIZE + 10}); // out of range + auto d = cudaq::qec::decoder::get("placement_probe_decoder", H, o); + int mode_before = -1, mode_after = -2; + bool threw_invalid = false, mask_restored = false; + int mode_in_decode = -1; + std::size_t sz = 0; + std::thread worker([&] { + syscall(SYS_get_mempolicy, &mode_before, nullptr, 0UL, nullptr, 0UL); + cpu_set_t before, after; + CPU_ZERO(&before); + CPU_ZERO(&after); + sched_getaffinity(0, sizeof(before), &before); + try { + d->bind_current_thread(); + } catch (const std::invalid_argument &) { + threw_invalid = true; + } + syscall(SYS_get_mempolicy, &mode_after, nullptr, 0UL, nullptr, 0UL); + sched_getaffinity(0, sizeof(after), &after); + mask_restored = CPU_EQUAL(&before, &after); + // Not bound => the guard must still fire on this thread's guarded decode. + cudaqx::tensor syndrome({2}); + auto r = d->decode(syndrome); + sz = r.result.size(); + if (sz == 4) + mode_in_decode = static_cast(r.result[3]); + }); + worker.join(); + EXPECT_TRUE(threw_invalid) + << "out-of-range cpu_affinity must throw std::invalid_argument"; + EXPECT_EQ(mode_after, mode_before) << "failed bind leaked a mempolicy change"; + EXPECT_TRUE(mask_restored) << "failed bind leaked a CPU-affinity change"; + ASSERT_EQ(sz, 4u); + EXPECT_EQ(mode_in_decode, MPOL_BIND) + << "decoder was left bound after a failed bind_current_thread: the " + "guard did not apply mempolicy on a later decode"; +#else + GTEST_SKIP() << "Linux-only"; +#endif +} + +// decode_on_pinned_thread must not steal the caller's binding: after a +// bind_current_thread() + decode_on_pinned_thread() sequence, a decode_batch() +// from the caller still skips the guard (the sentinel MPOL_DEFAULT set after +// the bind stays visible inside decode instead of the guard's MPOL_BIND). +TEST(DecoderAffinity, PinnedThreadDecodePreservesCallersBinding) { +#if defined(__linux__) + int probe = -1; + if (syscall(SYS_get_mempolicy, &probe, nullptr, 0UL, nullptr, 0UL) != 0) + GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map o; + o.insert("numa_node_id", 0); + o.insert("mempolicy", std::string("bind")); + auto d = cudaq::qec::decoder::get("placement_probe_decoder", H, o); + int pinned_mode = -1, batch_mode = -1; + std::size_t pinned_sz = 0, batch_n = 0, batch_sz = 0; + std::thread caller([&] { + d->bind_current_thread(); + // Sentinel on the caller thread; an applied guard would report MPOL_BIND. + syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); + auto rp = d->decode_on_pinned_thread( + std::vector{0.0f, 0.0f}); + pinned_sz = rp.result.size(); + if (pinned_sz == 4) + pinned_mode = static_cast(rp.result[3]); // pinned worker bound + auto rb = d->decode_batch( + std::vector>{{0.0f, 0.0f}}); + batch_n = rb.size(); + if (batch_n == 1) { + batch_sz = rb[0].result.size(); + if (batch_sz == 4) + batch_mode = static_cast(rb[0].result[3]); + } + }); + caller.join(); + ASSERT_EQ(pinned_sz, 4u); + EXPECT_EQ(pinned_mode, MPOL_BIND) + << "decode_on_pinned_thread's worker was not bound to the decoder's " + "mempolicy"; + ASSERT_EQ(batch_n, 1u); + ASSERT_EQ(batch_sz, 4u); + EXPECT_EQ(batch_mode, MPOL_DEFAULT) + << "caller's binding did not survive decode_on_pinned_thread: " + "decode_batch re-applied the guard on the bound caller"; +#else + GTEST_SKIP() << "Linux-only"; +#endif +} diff --git a/libs/qec/unittests/test_device_affinity.cpp b/libs/qec/unittests/test_device_affinity.cpp index 34d46b9ba..9744e41af 100644 --- a/libs/qec/unittests/test_device_affinity.cpp +++ b/libs/qec/unittests/test_device_affinity.cpp @@ -65,17 +65,25 @@ TEST(HardwareAffinity, MempolicyDefaultIsPreferredNotBind) { namespace da = cudaq::qec::detail_affinity; if (!mempolicy_syscalls_usable()) GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; + cpu_set_t saved; + CPU_ZERO(&saved); + sched_getaffinity(0, sizeof(saved), &saved); da::bind_this_thread_to_numa_node(0); EXPECT_EQ(da::current_thread_mempolicy_mode(), MPOL_PREFERRED); syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); + sched_setaffinity(0, sizeof(saved), &saved); } TEST(HardwareAffinity, MempolicyBindWhenRequested) { namespace da = cudaq::qec::detail_affinity; if (!mempolicy_syscalls_usable()) GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; - da::bind_this_thread_to_numa_node(0, da::mempolicy_mode::bind); + cpu_set_t saved; + CPU_ZERO(&saved); + sched_getaffinity(0, sizeof(saved), &saved); + da::bind_this_thread_to_numa_node(0, cudaq::qec::mempolicy_mode::bind); EXPECT_EQ(da::current_thread_mempolicy_mode(), MPOL_BIND); syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); + sched_setaffinity(0, sizeof(saved), &saved); } TEST(HardwareAffinity, MempolicyRejectsUnknownStringThrows) { cudaqx::heterogeneous_map p; @@ -153,8 +161,7 @@ TEST(HardwareAffinity, BindRegionSetsPolicyOnBuffer) { ASSERT_NE(p, nullptr); da::bind_region_to_numa_node(p, bytes, 0); // preferred, node 0 int mode = -1; - long rc = - syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, p, 1UL /*MPOL_F_ADDR*/); + long rc = syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, p, MPOL_F_ADDR); if (rc == 0) EXPECT_TRUE(mode == MPOL_PREFERRED || mode == MPOL_DEFAULT) << "region policy after preferred-bind should be PREFERRED (or DEFAULT " diff --git a/libs/qec/unittests/test_pinning_benchmark.cpp b/libs/qec/unittests/test_pinning_benchmark.cpp index 66d0df2d3..19c159e00 100644 --- a/libs/qec/unittests/test_pinning_benchmark.cpp +++ b/libs/qec/unittests/test_pinning_benchmark.cpp @@ -5,16 +5,29 @@ * This source code and the accompanying materials are made available under * * the terms of the Apache License 2.0 which accompanies this distribution. * ******************************************************************************/ +#include "support/thread_placement.h" #include "cudaq/qec/decoder.h" #include #include +#include #include +#include #include +#if defined(__linux__) +#include +#include +#include +#endif // Resolved from the LD_PRELOAD shim when preloaded; weak so the binary still // links (and the test cleanly skips) when it is not. extern "C" __attribute__((weak)) long cudaqx_affinity_syscall_count(); extern "C" __attribute__((weak)) void cudaqx_affinity_syscall_reset(); +// Per-syscall counters (same shim). +extern "C" __attribute__((weak)) long cudaqx_sched_setaffinity_count(); +extern "C" __attribute__((weak)) long cudaqx_sched_getaffinity_count(); +extern "C" __attribute__((weak)) long cudaqx_set_mempolicy_count(); +extern "C" __attribute__((weak)) long cudaqx_get_mempolicy_count(); namespace { cudaqx::tensor makeH() { @@ -30,6 +43,54 @@ std::unique_ptr makePinnedLut() { } const std::vector> kChunk = {{0.1, 0.1}, {0.2, 0.2}}; + +// ---- invariant-test helpers (shim counters + placement snapshots) ---------- +using cudaq::qec::test::thread_placement; + +bool shimCountersPresent() { + return cudaqx_affinity_syscall_reset && cudaqx_sched_setaffinity_count && + cudaqx_sched_getaffinity_count && cudaqx_set_mempolicy_count && + cudaqx_get_mempolicy_count; +} +struct ShimCounts { + long setaff, getaff, setmem, getmem; +}; +ShimCounts readShimCounts() { + return {cudaqx_sched_setaffinity_count(), cudaqx_sched_getaffinity_count(), + cudaqx_set_mempolicy_count(), cudaqx_get_mempolicy_count()}; +} + +// Pinned decoder with D/O set so ALL entry points (incl. enqueue_syndrome) +// are callable. +std::unique_ptr makeEntryReadyPinnedLut() { + auto d = makePinnedLut(); + d->set_D_sparse(std::vector>{{0}, {1}}); + d->set_O_sparse(std::vector>{{0, 1}}); + return d; +} + +// Every guarded entry point whose CUDA/NUMA guard runs on the CALLING thread +// (decode_async runs its guard on a worker thread; tested separately). +struct GuardedEntryPoint { + const char *name; + void (*call)(cudaq::qec::decoder &); +}; +const GuardedEntryPoint kCallerGuardedEntryPoints[] = { + {"decode_batch", [](cudaq::qec::decoder &d) { d.decode_batch(kChunk); }}, + {"decode_tensor", + [](cudaq::qec::decoder &d) { + cudaqx::tensor syndrome({2}); // zero-initialized, rank-1 + d.decode(syndrome); + }}, + {"enqueue_syndrome", + [](cudaq::qec::decoder &d) { + std::vector syndrome = {1, 1}; + d.enqueue_syndrome(syndrome); + }}, +}; +void callDecodeAsync(cudaq::qec::decoder &d) { + d.decode_async(kChunk[0]).get(); +} } // namespace // Deterministic gate: after binding, a decode loop must issue ZERO @@ -59,6 +120,56 @@ TEST(PinningBenchmark, BoundDecodeLoopIssuesNoAffinitySyscalls) { << "bound decode loop must not re-issue affinity syscalls"; } +TEST(PinningBenchmark, BindOnOneThreadStillGuardsAnotherThread) { + if (!cudaqx_affinity_syscall_count) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + auto d = makePinnedLut(); + d->bind_current_thread(); // bind on the main test thread + + long from_other_thread = -1; + std::thread other([&] { + cudaqx_affinity_syscall_reset(); + d->decode_batch(kChunk); + from_other_thread = cudaqx_affinity_syscall_count(); + }); + other.join(); + EXPECT_GT(from_other_thread, 0) + << "a thread that never called bind_current_thread() must still be " + "guarded, even though a DIFFERENT thread bound this decoder"; +} + +// Guards the decode_on_pinned_thread() contract: its one-shot worker thread +// binds itself for the duration of a single decode, but that binding must +// not stick to the decoder object once the worker exits. A subsequent +// decode_batch() from the (unbound) main thread must still be guarded. +TEST(PinningBenchmark, OneShotPinnedDecodeDoesNotStickToObject) { + if (!cudaqx_affinity_syscall_count) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + auto d = makePinnedLut(); + d->decode_on_pinned_thread(kChunk[0]); + + cudaqx_affinity_syscall_reset(); + d->decode_batch(kChunk); + EXPECT_GT(cudaqx_affinity_syscall_count(), 0) + << "decode_on_pinned_thread()'s one-shot worker binding must not " + "outlive it; a later decode_batch() from the main thread must " + "still issue affinity syscalls"; +} + +TEST(PinningBenchmark, EnqueueSyndromeAppliesGuardOnUnboundThread) { + if (!cudaqx_affinity_syscall_count) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + auto d = makePinnedLut(); + d->set_D_sparse(std::vector>{{0}, {1}}); + d->set_O_sparse(std::vector>{{0, 1}}); + cudaqx_affinity_syscall_reset(); + std::vector syndrome = {1, 1}; + d->enqueue_syndrome(syndrome); + EXPECT_GT(cudaqx_affinity_syscall_count(), 0) + << "enqueue_syndrome's internal decode() call must apply the NUMA " + "guard on an unbound thread, same as decode_batch()"; +} + // Loose A/B (report + gross-regression guard only; timing is noisy). TEST(PinningBenchmark, BoundThroughputNotWorseThanUnbound) { auto d = makePinnedLut(); @@ -82,3 +193,122 @@ TEST(PinningBenchmark, BoundThroughputNotWorseThanUnbound) { // Pinning removes per-call guard work; it must not be materially slower. EXPECT_LT(bound_us, unbound_us * 1.5); } + +TEST(PinningBenchmark, DecodeBatchRestoresExactPriorMempolicy) { +#if defined(__linux__) + int mode = -1; + if (syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL) != 0) + GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; + unsigned long nodemask = 1UL; // node 0 + if (syscall(SYS_set_mempolicy, MPOL_BIND, &nodemask, sizeof(nodemask) * 8) != + 0) + GTEST_SKIP() << "set_mempolicy unavailable (container seccomp?)"; + auto d = makePinnedLut(); // numa_node_id=0, unbound (per-call guard runs) + d->decode_batch(kChunk); + int mode_after = -1; + syscall(SYS_get_mempolicy, &mode_after, nullptr, 0UL, nullptr, 0UL); + syscall(SYS_set_mempolicy, MPOL_DEFAULT, nullptr, 0UL); // cleanup regardless + EXPECT_EQ(mode_after, MPOL_BIND) + << "decode_batch's guard must restore the exact prior mempolicy " + "(MPOL_BIND), not force MPOL_DEFAULT"; +#else + GTEST_SKIP() << "Linux-only"; +#endif +} + +// The un-restorable-policy scenario: a seccomp profile that blocks +// get_mempolicy while allowing set_mempolicy. A temporary guard that cannot +// capture the prior policy must not change it at all — it could never restore +// it, so the change would silently outlive the decode. The shim's +// CUDAQX_SHIM_FAIL_GET_MEMPOLICY injection simulates the blocked capture. +TEST(PinningInvariants, GuardSkipsMempolicyWhenCaptureFails) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; +#if defined(__linux__) + // Baseline read runs with injection off. + int before = -1; + if (syscall(SYS_get_mempolicy, &before, nullptr, 0UL, nullptr, 0UL) != 0) + GTEST_SKIP() << "mempolicy syscalls unavailable (container seccomp?)"; + + auto d = makePinnedLut(); // numa_node_id=0, unbound -> guard runs per call + setenv("CUDAQX_SHIM_FAIL_GET_MEMPOLICY", "1", 1); + d->decode_batch(kChunk); + unsetenv("CUDAQX_SHIM_FAIL_GET_MEMPOLICY"); + + int after = -1; + ASSERT_EQ(syscall(SYS_get_mempolicy, &after, nullptr, 0UL, nullptr, 0UL), 0); + EXPECT_EQ(after, before) + << "the guard applied a mempolicy it could not capture; the policy " + "leaked past the decode (capture-failure must skip set_mempolicy)"; +#else + GTEST_SKIP() << "Linux-only"; +#endif +} + +// Bound-path invariant, per entry point: after bind_current_thread() each +// caller-thread entry point must issue ZERO placement syscalls (all four +// counters) and leave the calling thread's placement bit-identical. +TEST(PinningInvariants, BoundEntryPointsIssueNoPlacementSyscalls) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + for (const auto &ep : kCallerGuardedEntryPoints) { + SCOPED_TRACE(ep.name); + auto d = makeEntryReadyPinnedLut(); + d->bind_current_thread(); + auto before = thread_placement::capture(); + cudaqx_affinity_syscall_reset(); + ep.call(*d); + auto c = readShimCounts(); + EXPECT_EQ(c.setaff, 0) << "bound path issued sched_setaffinity"; + EXPECT_EQ(c.getaff, 0) << "bound path issued sched_getaffinity"; + EXPECT_EQ(c.setmem, 0) << "bound path issued set_mempolicy"; + EXPECT_EQ(c.getmem, 0) << "bound path issued get_mempolicy"; + auto after = thread_placement::capture(); + EXPECT_EQ(before, after) << "bound path changed placement: " + << before.describe_difference(after); + } +} + +// decode_async decodes on a fresh std::async worker which can never be the +// bound thread, so its guard runs on the WORKER by design (asserted >0 here: +// deleting that guard fails this test). The caller-side invariant is that the +// calling thread's placement is untouched even though it holds the binding. +TEST(PinningInvariants, BoundDecodeAsyncGuardsWorkerAndLeavesCallerPlacement) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + auto d = makeEntryReadyPinnedLut(); + d->bind_current_thread(); + auto before = thread_placement::capture(); + cudaqx_affinity_syscall_reset(); + callDecodeAsync(*d); + EXPECT_GT(readShimCounts().setmem, 0) + << "decode_async's worker-thread guard must still attempt set_mempolicy " + "(the caller's binding must not leak to the worker)"; + auto after = thread_placement::capture(); + EXPECT_EQ(before, after) << "decode_async changed the CALLING thread's " + "placement: " + << before.describe_difference(after); +} + +// Unbound-path invariant, per entry point (incl. decode_async): the guard must +// actually run (set_mempolicy attempted -- deleting the guard fails this) AND +// fully restore the calling thread's placement afterwards. +TEST(PinningInvariants, UnboundEntryPointsApplyAndFullyRestoreGuard) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + std::vector entries(std::begin(kCallerGuardedEntryPoints), + std::end(kCallerGuardedEntryPoints)); + entries.push_back({"decode_async", callDecodeAsync}); + for (const auto &ep : entries) { + SCOPED_TRACE(ep.name); + auto d = makeEntryReadyPinnedLut(); // numa_node_id=0, never bound + auto before = thread_placement::capture(); + cudaqx_affinity_syscall_reset(); + ep.call(*d); + EXPECT_GT(readShimCounts().setmem, 0) + << "unbound path must attempt set_mempolicy (guard deleted?)"; + auto after = thread_placement::capture(); + EXPECT_EQ(before, after) << "guard applied but not fully restored: " + << before.describe_difference(after); + } +} From fd459112eb7eddca98601addc50fbc6a39e7ba42 Mon Sep 17 00:00:00 2001 From: kvmto Date: Mon, 6 Jul 2026 12:54:57 +0000 Subject: [PATCH 09/16] bug bash Signed-off-by: kvmto --- libs/qec/include/cudaq/qec/decoder.h | 2 + libs/qec/lib/decoder.cpp | 25 +++++++--- .../plugins/chromobius/chromobius.cpp | 3 ++ .../qec/lib/realtime/qec_realtime_session.cpp | 50 +++++++++++++------ libs/qec/unittests/test_pinning_benchmark.cpp | 39 +++++++++++++++ 5 files changed, 96 insertions(+), 23 deletions(-) diff --git a/libs/qec/include/cudaq/qec/decoder.h b/libs/qec/include/cudaq/qec/decoder.h index 50d65e4fc..b1f695928 100644 --- a/libs/qec/include/cudaq/qec/decoder.h +++ b/libs/qec/include/cudaq/qec/decoder.h @@ -197,6 +197,8 @@ class decoder /// value is the probability that the syndrome measurement is a |1>. /// @returns std::future of a vector of length `block_size` with soft /// probabilities of errors in each index. + /// @note The caller must ensure the decoder outlives the returned future. + /// Destroying the decoder before calling .get() is undefined behaviour. virtual std::future decode_async(const std::vector &syndrome); diff --git a/libs/qec/lib/decoder.cpp b/libs/qec/lib/decoder.cpp index 6c535d2b3..d231dc7bd 100644 --- a/libs/qec/lib/decoder.cpp +++ b/libs/qec/lib/decoder.cpp @@ -118,7 +118,8 @@ struct NumaGuard { ~NumaGuard() { if (mempol_set_) cudaq::qec::detail_affinity::restore_thread_mempolicy(prev_mempolicy_); - if (affinity_set_ && has_prev_affinity_) + // has_prev_affinity_ is implied: it is set iff affinity_set_ is set. + if (affinity_set_) if (sched_setaffinity(0, sizeof(prev_set_), &prev_set_) != 0) cudaq::qec::detail_affinity::affinity_warn( "NumaGuard restore: failed to reset thread affinity: " + @@ -360,9 +361,12 @@ decoder::decode_on_pinned_thread(const std::vector &syndrome) { std::memory_order_acq_rel); }); worker.join(); - // Restore the caller's prior binding that the worker overwrote. The worker - // has joined so this store races with nothing. - bound_thread_.store(caller_id, std::memory_order_release); + // Restore caller_id only if the slot is still empty (worker cleared it). + // If a concurrent bind_current_thread() wrote a different id between the + // worker's CAS-clear and here, leave that binding intact. + std::thread::id empty{}; + bound_thread_.compare_exchange_strong(empty, caller_id, + std::memory_order_acq_rel); if (err) std::rethrow_exception(err); return result; @@ -415,8 +419,10 @@ std::string decoder::get_version() const { std::future decoder::decode_async(const std::vector &syndrome) { - // Capture by value: avoids a data race if the decoder is destroyed before - // the future resolves. + // The three affinity scalars are captured by value so the lambda doesn't + // dereference decoder members on the hot path. + // LIFETIME PRECONDITION: the decoder must outlive the returned future. + // Destroying the decoder before calling .get() is undefined behaviour. const int cuda_id = cuda_device_id_; const int numa_id = numa_node_id_; const cudaq::qec::mempolicy_mode mempolicy = mempolicy_; @@ -463,7 +469,12 @@ decoder::get(const std::string &name, const decoder_init &init, if (!is_affinity_key(kv.first)) plugin_params.insert(kv.first, kv.second); auto d = iter->second(init, plugin_params); - d->set_hardware_params(param_map); + // Inject the pre-computed NUMA node so set_hardware_params() doesn't + // re-derive it via a second numa_node_for_cuda_device() sysfs read. + cudaqx::heterogeneous_map hw_params = param_map; + if (cudaq::qec::read_numa_node_id(param_map) < 0 && node >= 0) + hw_params.insert("numa_node_id", node); + d->set_hardware_params(hw_params); return d; } diff --git a/libs/qec/lib/decoders/plugins/chromobius/chromobius.cpp b/libs/qec/lib/decoders/plugins/chromobius/chromobius.cpp index 54712c57d..71222d8ca 100644 --- a/libs/qec/lib/decoders/plugins/chromobius/chromobius.cpp +++ b/libs/qec/lib/decoders/plugins/chromobius/chromobius.cpp @@ -140,6 +140,9 @@ class chromobius : public decoder { // observable-reduction logic treat each predicted bit as its own observable // correction. this->set_O_sparse(identity_sparse(num_observables)); + // Chromobius returns observable flips directly; declare this so the base + // class knows not to project through O_sparse a second time. + set_result_type(decode_to_obs); } decoder_result decode(const std::vector &syndrome) override { diff --git a/libs/qec/lib/realtime/qec_realtime_session.cpp b/libs/qec/lib/realtime/qec_realtime_session.cpp index 1bc1535d2..a2ab17700 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.cpp +++ b/libs/qec/lib/realtime/qec_realtime_session.cpp @@ -890,13 +890,24 @@ void qec_realtime_session::start_host_loop() { const int node = session_numa_node_; host_loop_thread_ = std::thread([this, node]() { - // Persistent bind: this thread does nothing but decode for its lifetime. - try { - cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(node); - } catch (const std::exception &e) { - cudaq::qec::detail_affinity::affinity_warn( - "host loop thread NUMA bind failed: " + std::string(e.what()) + - " — continuing without affinity"); + // Persistent bind via decoder::bind_current_thread() so that + // is_bound_here() returns true and per-call NUMA/CUDA guards are + // suppressed in the decode hot path. + // Guard: skip if initialize() detected a NUMA conflict and set + // session_numa_node_ < 0 — per-call guards must stay active then. + if (node >= 0) { + for (auto &d : decoders_) { + if (!d) + continue; + try { + d->bind_current_thread(); + } catch (const std::exception &e) { + cudaq::qec::detail_affinity::affinity_warn( + "host loop thread bind failed for decoder " + + std::to_string(d->get_decoder_id()) + ": " + + std::string(e.what()) + " — continuing without guard-skip"); + } + } } cudaq_host_dispatcher_loop(&host_ctx_); }); @@ -1009,15 +1020,22 @@ void qec_realtime_session::start_host_loop() { { const int node = session_numa_node_; host_loop_thread_ = std::thread([this, node]() { - // Mirror the HOST-mode dispatch thread's pinning above: the CPU-side - // monitor thread benefits from the same NUMA locality regardless of - // dispatch mode. - try { - cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(node); - } catch (const std::exception &e) { - cudaq::qec::detail_affinity::affinity_warn( - "host loop thread NUMA bind failed: " + std::string(e.what()) + - " — continuing without affinity"); + // Mirror HOST-mode: bind via decoder::bind_current_thread() so that + // guard-skip is activated for the decode hot path. + // Guard: skip if initialize() detected a NUMA conflict (node < 0). + if (node >= 0) { + for (auto &d : decoders_) { + if (!d) + continue; + try { + d->bind_current_thread(); + } catch (const std::exception &e) { + cudaq::qec::detail_affinity::affinity_warn( + "host loop thread bind failed for decoder " + + std::to_string(d->get_decoder_id()) + ": " + + std::string(e.what()) + " — continuing without guard-skip"); + } + } } cudaq_host_dispatcher_loop(&host_ctx_); }); diff --git a/libs/qec/unittests/test_pinning_benchmark.cpp b/libs/qec/unittests/test_pinning_benchmark.cpp index 19c159e00..b28657a88 100644 --- a/libs/qec/unittests/test_pinning_benchmark.cpp +++ b/libs/qec/unittests/test_pinning_benchmark.cpp @@ -156,6 +156,45 @@ TEST(PinningBenchmark, OneShotPinnedDecodeDoesNotStickToObject) { "still issue affinity syscalls"; } +// Verify that decode_on_pinned_thread() does not clobber a concurrent +// bind_current_thread() that runs from a third thread. +TEST(PinningBenchmark, OneShotDoesNotClobberConcurrentBind) { + if (!cudaqx_affinity_syscall_count) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + auto d = makePinnedLut(); + + // Thread B will call bind_current_thread() while the one-shot worker runs. + std::atomic b_bound{false}; + std::thread thread_b([&] { + // Spin until main_worker signals us to bind. + while (!b_bound.load(std::memory_order_acquire)) + std::this_thread::yield(); + d->bind_current_thread(); + }); + + // Run decode_on_pinned_thread from the main thread; signal B to bind midway. + std::thread main_worker([&] { + // Signal B just before decode so the race window is as wide as possible. + b_bound.store(true, std::memory_order_release); + d->decode_on_pinned_thread(kChunk[0]); + }); + + main_worker.join(); + thread_b.join(); + + // After both threads finish, whoever bound last should have their binding + // intact. At a minimum, is_bound_here() from thread_b must not permanently + // suppress guards: a subsequent decode from an unbound thread must still + // issue syscalls. + cudaqx_affinity_syscall_reset(); + // Run decode from a fresh thread that was never bound. + std::thread unbound([&] { d->decode_batch(kChunk); }); + unbound.join(); + // An unbound thread must always fire the guard. + EXPECT_GT(cudaqx_affinity_syscall_count(), 0) + << "guard must still fire for unbound threads after concurrent-bind race"; +} + TEST(PinningBenchmark, EnqueueSyndromeAppliesGuardOnUnboundThread) { if (!cudaqx_affinity_syscall_count) GTEST_SKIP() << "affinity-counter shim not preloaded"; From e25478a4991752779fc9c2dd45dd751a3914b937 Mon Sep 17 00:00:00 2001 From: kvmto Date: Mon, 6 Jul 2026 14:59:06 +0000 Subject: [PATCH 10/16] fix(qec): merge-readiness hardening for decoder hardware pinning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: placement-sensitive tests skip (not fail) where container seccomp blocks the mempolicy syscalls; persistent binds run on disposable worker threads so the gtest main thread stays clean; exact-core asserts probe the allowed cpuset first. Realtime session: HOST-mode ring buffers are page-aligned so the mbind(MPOL_MF_MOVE) migration actually succeeds (calloc pointers failed EINVAL with a misleading CAP_SYS_NICE warning); calloc's size-overflow protection restored; the dispatch thread registers via bind_current_thread() only when all decoders agree on cuda_device_id, numa_node_id, and cpu_affinity (disagreement keeps per-call guards active, with plain NUMA locality as fallback); new decoder::unbind_thread() clears registrations at stop_loops() so a recycled thread id can never silently skip the guards. Python: @qec.decoder-registered decoders get decode-time pinning — affinity kwargs are stripped before __init__ and stored on the C++ base (only the four affinity keys are converted; arbitrary kwargs pass through untouched); cuda_device_id()/numa_node_id() accessors exposed; read_cpu_affinity accepts Python-shaped double lists. Hygiene: license header on the affinity syscall shim; reserved identifiers _RestoreDevice/_ScopedNuma renamed. Pinning A/B (5000 decode_batch calls): bound 516.6us vs unbound 59301.4us. Signed-off-by: kvmto --- libs/qec/include/cudaq/qec/decoder.h | 11 + libs/qec/include/cudaq/qec/device_affinity.h | 18 +- libs/qec/lib/decoder.cpp | 4 + .../plugins/trt_decoder/trt_decoder.cpp | 10 +- .../qec/lib/realtime/qec_realtime_session.cpp | 136 ++++++++--- libs/qec/lib/realtime/qec_realtime_session.h | 6 + libs/qec/python/bindings/py_decoder.cpp | 57 ++++- libs/qec/python/tests/test_decoder.py | 55 +++++ .../support/affinity_syscall_shim.cpp | 7 + libs/qec/unittests/test_decoders.cpp | 30 ++- libs/qec/unittests/test_device_affinity.cpp | 37 ++- libs/qec/unittests/test_pinning_benchmark.cpp | 224 ++++++++++++------ 12 files changed, 457 insertions(+), 138 deletions(-) diff --git a/libs/qec/include/cudaq/qec/decoder.h b/libs/qec/include/cudaq/qec/decoder.h index b1f695928..0958e6bda 100644 --- a/libs/qec/include/cudaq/qec/decoder.h +++ b/libs/qec/include/cudaq/qec/decoder.h @@ -268,6 +268,9 @@ class decoder /// @brief Target CUDA device for this decoder (-1 = inherit caller's device). int cuda_device_id() const { return cuda_device_id_; } + /// @brief Explicit CPU list for this decoder (empty = derive from NUMA node). + const std::vector &cpu_affinity() const { return cpu_affinity_; } + /// @brief Persistently bind the CALLING thread to this decoder's NUMA node /// (and CUDA device). Call once from the thread that will own decode()/ /// enqueue for this decoder (e.g. a realtime worker). No restore. @@ -281,6 +284,14 @@ class decoder /// throughput drive decode on a long-lived bound thread instead. decoder_result decode_on_pinned_thread(const std::vector &syndrome); + /// @brief Forget any bind_current_thread() registration on this decoder so + /// guarded entry points stop skipping their per-call guard. Call before the + /// bound thread exits (e.g. session teardown): thread ids are recycled by + /// the runtime, and a stale registration would let an unrelated new thread + /// silently skip the guard. Does not undo the OS-level placement of the + /// (exiting) bound thread. + void unbind_thread(); + // -- Begin realtime decoding API -- // Note: all of the current realtime decoding API is designed to be used with diff --git a/libs/qec/include/cudaq/qec/device_affinity.h b/libs/qec/include/cudaq/qec/device_affinity.h index 1c6a012c6..3cf24f841 100644 --- a/libs/qec/include/cudaq/qec/device_affinity.h +++ b/libs/qec/include/cudaq/qec/device_affinity.h @@ -58,12 +58,26 @@ inline mempolicy_mode read_mempolicy(const cudaqx::heterogeneous_map ¶ms) { } /// @brief Read "cpu_affinity": a list of CPU core ids. Absent -> empty (no -/// override). +/// override). Accepts vector (C++/YAML) and vector with integral +/// values (Python kwargs path — lists arrive as doubles). inline std::vector read_cpu_affinity(const cudaqx::heterogeneous_map ¶ms) { if (!params.contains("cpu_affinity")) return {}; - return params.get>("cpu_affinity"); + try { + return params.get>("cpu_affinity"); + } catch (...) { + } + const auto vals = params.get>("cpu_affinity"); + std::vector cores; + cores.reserve(vals.size()); + for (double v : vals) { + if (v != static_cast(static_cast(v))) + throw std::runtime_error("cpu_affinity core ids must be integers (got " + + std::to_string(v) + ")"); + cores.push_back(static_cast(v)); + } + return cores; } /// @brief NUMA node local to a CUDA device (via PCIe locality), or -1 if the diff --git a/libs/qec/lib/decoder.cpp b/libs/qec/lib/decoder.cpp index d231dc7bd..57f744ea4 100644 --- a/libs/qec/lib/decoder.cpp +++ b/libs/qec/lib/decoder.cpp @@ -340,6 +340,10 @@ bool decoder::is_bound_here() const { std::this_thread::get_id(); } +void decoder::unbind_thread() { + bound_thread_.store(std::thread::id{}, std::memory_order_release); +} + decoder_result decoder::decode_on_pinned_thread(const std::vector &syndrome) { decoder_result result; diff --git a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp index 9173f8c30..6a1bb3d41 100644 --- a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp +++ b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp @@ -917,10 +917,10 @@ trt_decoder::decode_batch(const std::vector> &syndromes) { _dev_switched = true; } } - struct _RestoreDevice { + struct RestoreDevice { int prev; bool active; - ~_RestoreDevice() { + ~RestoreDevice() { if (!active) return; if (cudaSetDevice(prev) != cudaSuccess) @@ -931,14 +931,14 @@ trt_decoder::decode_batch(const std::vector> &syndromes) { } _dev_guard{_prev_dev, _dev_switched}; namespace da = cudaq::qec::detail_affinity; - struct _ScopedNuma { + struct ScopedNuma { bool active = false; bool has_prev_affinity = false; #if defined(__linux__) cpu_set_t prev_set{}; #endif da::mempolicy_state prev_mempolicy; - _ScopedNuma(int node, cudaq::qec::mempolicy_mode mode) { + ScopedNuma(int node, cudaq::qec::mempolicy_mode mode) { if (node < 0) return; bool apply_mempol = true; @@ -958,7 +958,7 @@ trt_decoder::decode_batch(const std::vector> &syndromes) { // syscall for out-of-range nodes, so there is nothing to undo then. active = true; } - ~_ScopedNuma() { + ~ScopedNuma() { if (!active) return; #if defined(__linux__) diff --git a/libs/qec/lib/realtime/qec_realtime_session.cpp b/libs/qec/lib/realtime/qec_realtime_session.cpp index a2ab17700..5004ee47b 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.cpp +++ b/libs/qec/lib/realtime/qec_realtime_session.cpp @@ -22,8 +22,10 @@ #include #include #include +#include #include #include +#include namespace cudaq::qec::realtime { @@ -336,16 +338,37 @@ void qec_realtime_session::initialize() { { int chosen = -1; bool conflict = false; + int chosen_dev = -1; + bool dev_conflict = false; + std::vector chosen_cpus; + bool affinity_conflict = false; for (const auto &d : decoders_) { if (!d) continue; const int n = d->numa_node_id(); - if (n < 0) - continue; - if (chosen < 0) - chosen = n; - else if (chosen != n) - conflict = true; + if (n >= 0) { + if (chosen < 0) + chosen = n; + else if (chosen != n) + conflict = true; + } + const int dev = d->cuda_device_id(); + if (dev >= 0) { + if (chosen_dev < 0) + chosen_dev = dev; + else if (chosen_dev != dev) + dev_conflict = true; + } + // bind_current_thread() also applies each decoder's explicit + // cpu_affinity list persistently; disagreeing non-empty lists would + // silently resolve to last-bind-wins. Empty lists never disagree. + const std::vector &cpus = d->cpu_affinity(); + if (!cpus.empty()) { + if (chosen_cpus.empty()) + chosen_cpus = cpus; + else if (chosen_cpus != cpus) + affinity_conflict = true; + } } if (conflict) { CUDA_QEC_WARN( @@ -356,6 +379,19 @@ void qec_realtime_session::initialize() { } else { session_numa_node_ = chosen; } + if (dev_conflict) + CUDA_QEC_WARN( + "realtime decoders request different cuda_device_ids; the shared " + "dispatch thread cannot hold one device for all of them. " + "Guard-skip binding disabled; per-call guards stay active."); + if (affinity_conflict) + CUDA_QEC_WARN( + "realtime decoders request different cpu_affinity lists; the " + "shared dispatch thread can honor only one. Guard-skip binding " + "disabled; per-call guards stay active."); + bind_decoders_to_host_loop_ = !dev_conflict && !conflict && + !affinity_conflict && + (chosen >= 0 || chosen_dev >= 0); } allocate_ring_buffer(); populate_function_table(); @@ -618,29 +654,41 @@ void qec_realtime_session::allocate_ring_buffer() { shutdown_flag_dev_ = static_cast(d); } } else { - // HOST mode: plain host memory; the device-visible pointers alias the host - // backings (no GPU required at runtime). The host loop reads only the - // *_host views; the producer's address-as-flag publish uses rx_data_dev() - // (== rx_data_host_ here), which the host loop dereferences as host memory. - auto alloc_u64 = [&](volatile std::uint64_t *&host, - volatile std::uint64_t *&dev, const char *what) { - void *p = std::calloc(num_slots_, sizeof(std::uint64_t)); + // HOST mode: plain page-aligned host memory; the device-visible pointers + // alias the host backings (no GPU required at runtime). Page alignment is + // required by the mbind(2) migration below — a calloc pointer is not + // page-aligned and made the migration fail with EINVAL. + const std::size_t page = static_cast(sysconf(_SC_PAGESIZE)); + // calloc (the previous allocator here) detected multiplication overflow; + // aligned_alloc does not, so guard the num_slots_ * size products. + auto checked_mul = [](std::size_t a, std::size_t b) -> std::size_t { + if (b != 0 && a > std::numeric_limits::max() / b) + throw std::runtime_error( + "qec_realtime_session::initialize: ring size overflow"); + return a * b; + }; + auto alloc_zeroed_pages = [&](std::size_t bytes, + const char *what) -> void * { + const std::size_t rounded = (bytes + page - 1) & ~(page - 1); + void *p = std::aligned_alloc(page, rounded); if (!p) throw std::runtime_error( std::string( "qec_realtime_session::initialize: failed to allocate ") + what); + std::memset(p, 0, rounded); + return p; + }; + auto alloc_u64 = [&](volatile std::uint64_t *&host, + volatile std::uint64_t *&dev, const char *what) { + void *p = alloc_zeroed_pages( + checked_mul(num_slots_, sizeof(std::uint64_t)), what); host = static_cast(p); dev = host; }; auto alloc_u8 = [&](std::uint8_t *&host, std::uint8_t *&dev, const char *what) { - void *p = std::calloc(num_slots_, slot_size_); - if (!p) - throw std::runtime_error( - std::string( - "qec_realtime_session::initialize: failed to allocate ") + - what); + void *p = alloc_zeroed_pages(checked_mul(num_slots_, slot_size_), what); host = static_cast(p); dev = host; }; @@ -649,8 +697,9 @@ void qec_realtime_session::allocate_ring_buffer() { alloc_u8(rx_data_host_, rx_data_dev_, "RX ring data"); alloc_u8(tx_data_host_, tx_data_dev_, "TX ring data"); // Migrate the syndrome ring buffers onto the session node so the (pinned) - // dispatch thread reads them on-node. calloc already faulted these on the - // setup thread, so this relocates the pages (MPOL_MF_MOVE). + // dispatch thread reads them on-node. The pages are zero-filled (faulted) + // at allocation, so MPOL_MF_MOVE relocates them; unfaulted pages follow + // the VMA policy set by mbind. if (session_numa_node_ >= 0) { using cudaq::qec::detail_affinity::bind_region_to_numa_node; bind_region_to_numa_node((void *)rx_flags_host_, @@ -889,13 +938,14 @@ void qec_realtime_session::start_host_loop() { shutdown_flag_ = 0; const int node = session_numa_node_; - host_loop_thread_ = std::thread([this, node]() { + const bool bind_decoders = bind_decoders_to_host_loop_; + host_loop_thread_ = std::thread([this, node, bind_decoders]() { // Persistent bind via decoder::bind_current_thread() so that // is_bound_here() returns true and per-call NUMA/CUDA guards are - // suppressed in the decode hot path. - // Guard: skip if initialize() detected a NUMA conflict and set - // session_numa_node_ < 0 — per-call guards must stay active then. - if (node >= 0) { + // suppressed in the decode hot path. Only when initialize() proved all + // decoders agree on both placement knobs; otherwise keep per-call + // guards active and give the thread plain NUMA locality at most. + if (bind_decoders) { for (auto &d : decoders_) { if (!d) continue; @@ -908,6 +958,14 @@ void qec_realtime_session::start_host_loop() { std::string(e.what()) + " — continuing without guard-skip"); } } + } else if (node >= 0) { + try { + cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(node); + } catch (const std::exception &e) { + cudaq::qec::detail_affinity::affinity_warn( + "host loop thread NUMA bind failed: " + std::string(e.what()) + + " — continuing without affinity"); + } } cudaq_host_dispatcher_loop(&host_ctx_); }); @@ -1019,11 +1077,14 @@ void qec_realtime_session::start_host_loop() { { const int node = session_numa_node_; - host_loop_thread_ = std::thread([this, node]() { + const bool bind_decoders = bind_decoders_to_host_loop_; + host_loop_thread_ = std::thread([this, node, bind_decoders]() { // Mirror HOST-mode: bind via decoder::bind_current_thread() so that - // guard-skip is activated for the decode hot path. - // Guard: skip if initialize() detected a NUMA conflict (node < 0). - if (node >= 0) { + // guard-skip is activated for the decode hot path. Only when + // initialize() proved all decoders agree on both placement knobs; + // otherwise keep per-call guards active and give the thread plain NUMA + // locality at most. + if (bind_decoders) { for (auto &d : decoders_) { if (!d) continue; @@ -1036,6 +1097,14 @@ void qec_realtime_session::start_host_loop() { std::string(e.what()) + " — continuing without guard-skip"); } } + } else if (node >= 0) { + try { + cudaq::qec::detail_affinity::bind_this_thread_to_numa_node(node); + } catch (const std::exception &e) { + cudaq::qec::detail_affinity::affinity_warn( + "host loop thread NUMA bind failed: " + std::string(e.what()) + + " — continuing without affinity"); + } } cudaq_host_dispatcher_loop(&host_ctx_); }); @@ -1062,6 +1131,13 @@ void qec_realtime_session::stop_loops() { if (host_loop_thread_.joinable()) host_loop_thread_.join(); + // The dispatch thread may have registered itself on the decoders via + // bind_current_thread(); it is gone now — clear the registrations before + // its thread id can be recycled by an unrelated new thread. + for (auto &d : decoders_) + if (d) + d->unbind_thread(); + if (device_dispatcher_) { cudaq_dispatcher_stop(device_dispatcher_); cudaq_dispatcher_destroy(device_dispatcher_); diff --git a/libs/qec/lib/realtime/qec_realtime_session.h b/libs/qec/lib/realtime/qec_realtime_session.h index 62661e47b..1a1084ce6 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.h +++ b/libs/qec/lib/realtime/qec_realtime_session.h @@ -211,6 +211,12 @@ class __attribute__((visibility("default"))) qec_realtime_session { /// Single NUMA node this session pins to (dispatch thread + ring buffers). /// -1 = no pinning (unset, or decoders disagree — see initialize()). int session_numa_node_ = -1; + /// True when every decoder that sets a placement knob agrees on BOTH + /// cuda_device_id and numa_node_id — only then may the shared dispatch + /// thread register itself via bind_current_thread() (a bound decoder skips + /// its per-call guard, so binding under a device conflict would decode on + /// the wrong GPU). + bool bind_decoders_to_host_loop_ = false; std::uint64_t host_stats_counter_ = 0; // Plain (non-pinned) shutdown flag for HOST mode (no device kernel shares // it). diff --git a/libs/qec/python/bindings/py_decoder.cpp b/libs/qec/python/bindings/py_decoder.cpp index 69af10ad9..ce091bc51 100644 --- a/libs/qec/python/bindings/py_decoder.cpp +++ b/libs/qec/python/bindings/py_decoder.cpp @@ -251,6 +251,54 @@ std::unordered_map(item.first))) + affinity_kwargs[item.first] = item.second; + else + stripped[item.first] = item.second; + } + nb::object obj = PyDecoderRegistry::get_decoder( + name, H, nb::borrow(stripped.ptr())); + // No affinity kwargs: preserve exact pre-existing behavior (including BYOD + // classes that never call qec.Decoder.__init__). + if (affinity_kwargs.size() == 0) + return obj; + // Narrow try: only the base-class cast maps to the "uninitialized base" + // message. hetMapFromKwargs can itself throw nb::cast_error (e.g. a negative + // Python int for cuda_device_id) and must not be misreported as a missing + // qec.Decoder.__init__ call. + decoder *base = nullptr; + try { + base = &nb::cast(obj); + } catch (const nb::cast_error &) { + throw std::runtime_error( + "Python decoder '" + name + + "' did not initialize its qec.Decoder base; call " + "qec.Decoder.__init__(self, H) in __init__ before using hardware " + "affinity kwargs (cuda_device_id/numa_node_id/mempolicy/cpu_affinity)"); + } + base->set_hardware_params( + hetMapFromKwargs(nb::borrow(affinity_kwargs.ptr()))); + return obj; +} struct batch_decoder_result { // Python-facing constructor for decoder plugin authors. nanobind enforces @@ -705,6 +753,11 @@ void bindDecoder(nb::module_ &mod) { }, "Decode multiple syndromes and return the results", nb::arg("syndrome")) + .def("cuda_device_id", &decoder::cuda_device_id, + "Target CUDA device for this decoder (-1 = inherit caller's " + "device).") + .def("numa_node_id", &decoder::numa_node_id, + "Target NUMA node for this decoder (-1 = no binding).") .def("get_block_size", &decoder::get_block_size, "Get the size of the code block") .def("get_syndrome_size", &decoder::get_syndrome_size, @@ -870,7 +923,7 @@ void bindDecoder(nb::module_ &mod) { options["error_rate_vec"] = copyToPyArray(*defaults.error_rate_vec); nb::object H_obj = copyToPyArray(dem.detector_error_matrix); - return PyDecoderRegistry::get_decoder(name, H_obj, options); + return get_py_registered_decoder(name, H_obj, options); } return get_decoder(name, decoder_init{dem_text}, hetMapFromKwargs(options)); @@ -887,7 +940,7 @@ void bindDecoder(nb::module_ &mod) { } if (PyDecoderRegistry::contains(name)) { - return PyDecoderRegistry::get_decoder(name, H, options); + return get_py_registered_decoder(name, H, options); } cudaq::qec::sparse_binary_matrix H_sparse; diff --git a/libs/qec/python/tests/test_decoder.py b/libs/qec/python/tests/test_decoder.py index 3201de1c6..ac991f07a 100644 --- a/libs/qec/python/tests/test_decoder.py +++ b/libs/qec/python/tests/test_decoder.py @@ -1019,5 +1019,60 @@ def test_get_decoder_stim_dem_without_observables_returns_errors(): assert list(result.result) == [1.0] +def test_python_decoder_affinity_kwargs_stripped_and_stored(): + + @qec.decoder("strict_affinity_byod") + class StrictAffinityDecoder: + # Deliberately NO **kwargs: affinity keys must be stripped before + # __init__, mirroring the C++ decoder::get() contract. + def __init__(self, H): + qec.Decoder.__init__(self, H) + + def decode(self, syndrome): + r = qec.DecoderResult() + r.converged = True + r.result = [0.0, 0.0, 0.0] + return r + + H = np.array([[1, 1, 0], [0, 1, 1]], dtype=np.uint8) + # numa_node_id must not reach __init__ (no TypeError) and must be stored + # on the C++ base so guarded entry points pin correctly. + d = qec.get_decoder("strict_affinity_byod", H, numa_node_id=0) + assert d.numa_node_id() == 0 + assert d.cuda_device_id() == -1 + results = d.decode_batch([[0.0, 0.0], [0.1, 0.1]]) + assert len(results) == 2 + + +def test_python_decoder_affinity_kwargs_coexist_with_arbitrary_kwargs(): + received = {} + + @qec.decoder("affinity_plus_callback_byod") + class AffinityPlusCallbackDecoder: + + def __init__(self, H, **kwargs): + qec.Decoder.__init__(self, H) + received.update(kwargs) + + def decode(self, syndrome): + r = qec.DecoderResult() + r.converged = True + r.result = [0.0, 0.0, 0.0] + return r + + H = np.array([[1, 1, 0], [0, 1, 1]], dtype=np.uint8) + cb = lambda x: x + # Non-affinity kwargs of arbitrary Python types (e.g. callables) must pass + # through to __init__ untouched — only the affinity keys are converted to + # the C++ heterogeneous map, so this must construct without raising. + d = qec.get_decoder("affinity_plus_callback_byod", + H, + numa_node_id=0, + callback=cb) + assert d.numa_node_id() == 0 + assert received["callback"] is cb + assert "numa_node_id" not in received + + if __name__ == "__main__": pytest.main() diff --git a/libs/qec/unittests/support/affinity_syscall_shim.cpp b/libs/qec/unittests/support/affinity_syscall_shim.cpp index 47076c4fd..926c5c564 100644 --- a/libs/qec/unittests/support/affinity_syscall_shim.cpp +++ b/libs/qec/unittests/support/affinity_syscall_shim.cpp @@ -1,3 +1,10 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ // Test-only LD_PRELOAD interposer: counts the placement-related syscalls so a // test can assert a bound decode loop issues none (and an unbound one does). // Counts four syscalls: sched_setaffinity / sched_getaffinity (glibc symbols) diff --git a/libs/qec/unittests/test_decoders.cpp b/libs/qec/unittests/test_decoders.cpp index 0e8438aea..d8d479528 100644 --- a/libs/qec/unittests/test_decoders.cpp +++ b/libs/qec/unittests/test_decoders.cpp @@ -1045,19 +1045,23 @@ TEST(HardwarePinning, DecodeBatchLeavesPersistentDeviceBinding) { if (count < 2) GTEST_SKIP() << "needs >= 2 GPUs"; - cudaSetDevice(0); - cudaqx::tensor H({2, 3}); - cudaqx::heterogeneous_map params; - params.insert("cuda_device_id", 1); - auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); - - d->bind_current_thread(); // caller now persistently on device 1 - std::vector> batch = {{0.1f, 0.1f}}; - (void)d->decode_batch(batch); - - int dev = -1; - cudaGetDevice(&dev); - EXPECT_EQ(dev, 1) << "decode_batch restored device despite persistent bind"; + // Persistent bind must not leak into later tests: run on a worker thread. + std::thread t([] { + cudaSetDevice(0); + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + params.insert("cuda_device_id", 1); + auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); + + d->bind_current_thread(); // caller now persistently on device 1 + std::vector> batch = {{0.1f, 0.1f}}; + (void)d->decode_batch(batch); + + int dev = -1; + cudaGetDevice(&dev); + EXPECT_EQ(dev, 1) << "decode_batch restored device despite persistent bind"; + }); + t.join(); } TEST(HardwarePinning, BindCurrentThreadAppliesCpuAffinity) { diff --git a/libs/qec/unittests/test_device_affinity.cpp b/libs/qec/unittests/test_device_affinity.cpp index 9744e41af..e0916ab09 100644 --- a/libs/qec/unittests/test_device_affinity.cpp +++ b/libs/qec/unittests/test_device_affinity.cpp @@ -9,6 +9,7 @@ #include "cuda-qx/core/heterogeneous_map.h" #include "cudaq/qec/device_affinity.h" #include +#include #include #if defined(__linux__) @@ -50,6 +51,16 @@ TEST(DeviceAffinity, ReadNegativeThrows) { EXPECT_THROW(read_cuda_device_id(m), std::runtime_error); } +TEST(DeviceAffinity, ReadCpuAffinityAcceptsDoubleStorage) { + // Python kwargs deliver lists as vector; integral values convert. + heterogeneous_map m; + m.insert("cpu_affinity", std::vector{0.0, 2.0}); + EXPECT_EQ(cudaq::qec::read_cpu_affinity(m), (std::vector{0, 2})); + heterogeneous_map bad; + bad.insert("cpu_affinity", std::vector{0.5}); + EXPECT_THROW(cudaq::qec::read_cpu_affinity(bad), std::runtime_error); +} + #if defined(__linux__) // Container runtimes commonly block the mempolicy syscalls (seccomp without // CAP_SYS_NICE). The library degrades gracefully there (warns, keeps going), @@ -98,7 +109,11 @@ TEST(HardwareAffinity, CpuAffinityPinsToExactCores) { namespace da = cudaq::qec::detail_affinity; cpu_set_t saved; CPU_ZERO(&saved); - sched_getaffinity(0, sizeof(saved), &saved); + ASSERT_EQ(sched_getaffinity(0, sizeof(saved), &saved), 0); + // sched_setaffinity succeeds with the INTERSECTION of the requested and + // allowed masks; assert exact placement only where both cores are allowed. + if (!CPU_ISSET(0, &saved) || !CPU_ISSET(2, &saved)) + GTEST_SKIP() << "CPUs 0 and 2 not both in this process's allowed cpuset"; da::set_thread_cpu_affinity({0, 2}); auto cpus = da::current_thread_cpuset(); EXPECT_EQ(cpus, (std::vector{0, 2})); @@ -156,18 +171,20 @@ TEST(HardwareAffinity, NegativeNodeIsNoop) { } TEST(HardwareAffinity, BindRegionSetsPolicyOnBuffer) { namespace da = cudaq::qec::detail_affinity; - const size_t bytes = 4096; - void *p = std::calloc(1, bytes); + const long page = sysconf(_SC_PAGESIZE); + void *p = std::aligned_alloc(page, static_cast(page)); ASSERT_NE(p, nullptr); - da::bind_region_to_numa_node(p, bytes, 0); // preferred, node 0 + std::memset(p, 0, static_cast(page)); + da::bind_region_to_numa_node(p, static_cast(page), 0); // preferred int mode = -1; long rc = syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, p, MPOL_F_ADDR); - if (rc == 0) - EXPECT_TRUE(mode == MPOL_PREFERRED || mode == MPOL_DEFAULT) - << "region policy after preferred-bind should be PREFERRED (or DEFAULT " - "if unsupported)"; - da::bind_region_to_numa_node(p, bytes, - -1); // negative node -> no-op, no crash + if (rc != 0) { + std::free(p); + GTEST_SKIP() << "get_mempolicy(MPOL_F_ADDR) unavailable (seccomp?)"; + } + EXPECT_EQ(mode, MPOL_PREFERRED) + << "mbind on a page-aligned region must set MPOL_PREFERRED"; + da::bind_region_to_numa_node(p, static_cast(page), -1); // no-op ok std::free(p); } TEST(HardwareAffinity, BindRegionNode64Throws) { diff --git a/libs/qec/unittests/test_pinning_benchmark.cpp b/libs/qec/unittests/test_pinning_benchmark.cpp index b28657a88..5f3c1f1d7 100644 --- a/libs/qec/unittests/test_pinning_benchmark.cpp +++ b/libs/qec/unittests/test_pinning_benchmark.cpp @@ -60,6 +60,38 @@ ShimCounts readShimCounts() { cudaqx_set_mempolicy_count(), cudaqx_get_mempolicy_count()}; } +#if defined(__linux__) +// Container seccomp commonly blocks the mempolicy syscalls. When +// SYS_get_mempolicy is blocked, NumaGuard cannot capture the prior policy and +// correctly skips set_mempolicy entirely, so tests asserting setmem > 0 must +// skip, not fail (mirrors test_device_affinity.cpp). Read-only: no side +// effects on the thread's policy. +bool get_mempolicy_usable() { + int mode = -1; + return syscall(SYS_get_mempolicy, &mode, nullptr, 0UL, nullptr, 0UL) == 0; +} +#else +bool get_mempolicy_usable() { return false; } +#endif + +// bind_current_thread() is persistent (no restore). Run test bodies that bind +// on a disposable worker so the gtest main thread's placement stays intact +// for later tests. EXPECT_* record correctly from any thread. +template +void run_on_worker(F &&fn) { + std::exception_ptr err; + std::thread t([&] { + try { + fn(); + } catch (...) { + err = std::current_exception(); + } + }); + t.join(); + if (err) + std::rethrow_exception(err); +} + // Pinned decoder with D/O set so ALL entry points (incl. enqueue_syndrome) // are callable. std::unique_ptr makeEntryReadyPinnedLut() { @@ -99,43 +131,49 @@ void callDecodeAsync(cudaq::qec::decoder &d) { TEST(PinningBenchmark, BoundDecodeLoopIssuesNoAffinitySyscalls) { if (!cudaqx_affinity_syscall_count) GTEST_SKIP() << "affinity-counter shim not preloaded"; - auto d = makePinnedLut(); - constexpr int N = 200; + run_on_worker( + [] { + auto d = makePinnedLut(); + constexpr int N = 200; - // Canary: unbound -> per-call guard fires. - cudaqx_affinity_syscall_reset(); - for (int i = 0; i < N; ++i) - d->decode_batch(kChunk); - long unbound = cudaqx_affinity_syscall_count(); - EXPECT_GT(unbound, 0) << "unbound decode loop should issue affinity syscalls " - "(else the gate is vacuous)"; + // Canary: unbound -> per-call guard fires. + cudaqx_affinity_syscall_reset(); + for (int i = 0; i < N; ++i) + d->decode_batch(kChunk); + long unbound = cudaqx_affinity_syscall_count(); + EXPECT_GT(unbound, 0) + << "unbound decode loop should issue affinity syscalls " + "(else the gate is vacuous)"; - // Bound: bind once, then the loop must add none. - d->bind_current_thread(); - cudaqx_affinity_syscall_reset(); - for (int i = 0; i < N; ++i) - d->decode_batch(kChunk); - long bound = cudaqx_affinity_syscall_count(); - EXPECT_EQ(bound, 0) - << "bound decode loop must not re-issue affinity syscalls"; + // Bound: bind once, then the loop must add none. + d->bind_current_thread(); + cudaqx_affinity_syscall_reset(); + for (int i = 0; i < N; ++i) + d->decode_batch(kChunk); + long bound = cudaqx_affinity_syscall_count(); + EXPECT_EQ(bound, 0) + << "bound decode loop must not re-issue affinity syscalls"; + }); } TEST(PinningBenchmark, BindOnOneThreadStillGuardsAnotherThread) { if (!cudaqx_affinity_syscall_count) GTEST_SKIP() << "affinity-counter shim not preloaded"; - auto d = makePinnedLut(); - d->bind_current_thread(); // bind on the main test thread + run_on_worker([] { + auto d = makePinnedLut(); + d->bind_current_thread(); // bind on the binding worker thread - long from_other_thread = -1; - std::thread other([&] { - cudaqx_affinity_syscall_reset(); - d->decode_batch(kChunk); - from_other_thread = cudaqx_affinity_syscall_count(); + long from_other_thread = -1; + std::thread other([&] { + cudaqx_affinity_syscall_reset(); + d->decode_batch(kChunk); + from_other_thread = cudaqx_affinity_syscall_count(); + }); + other.join(); + EXPECT_GT(from_other_thread, 0) + << "a thread that never called bind_current_thread() must still be " + "guarded, even though a DIFFERENT thread bound this decoder"; }); - other.join(); - EXPECT_GT(from_other_thread, 0) - << "a thread that never called bind_current_thread() must still be " - "guarded, even though a DIFFERENT thread bound this decoder"; } // Guards the decode_on_pinned_thread() contract: its one-shot worker thread @@ -211,26 +249,28 @@ TEST(PinningBenchmark, EnqueueSyndromeAppliesGuardOnUnboundThread) { // Loose A/B (report + gross-regression guard only; timing is noisy). TEST(PinningBenchmark, BoundThroughputNotWorseThanUnbound) { - auto d = makePinnedLut(); - constexpr int N = 5000; - auto run = [&] { - auto t0 = std::chrono::steady_clock::now(); - for (int i = 0; i < N; ++i) - d->decode_batch(kChunk); - return std::chrono::duration( - std::chrono::steady_clock::now() - t0) - .count(); - }; - run(); // warm up - double unbound_us = run(); - d->bind_current_thread(); - run(); // warm up bound - double bound_us = run(); - std::printf( - "[pinning A/B] unbound=%.1fus bound=%.1fus (%d decodes) ratio=%.2f\n", - unbound_us, bound_us, N, bound_us / unbound_us); - // Pinning removes per-call guard work; it must not be materially slower. - EXPECT_LT(bound_us, unbound_us * 1.5); + run_on_worker([] { + auto d = makePinnedLut(); + constexpr int N = 5000; + auto run = [&] { + auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < N; ++i) + d->decode_batch(kChunk); + return std::chrono::duration( + std::chrono::steady_clock::now() - t0) + .count(); + }; + run(); // warm up + double unbound_us = run(); + d->bind_current_thread(); + run(); // warm up bound + double bound_us = run(); + std::printf( + "[pinning A/B] unbound=%.1fus bound=%.1fus (%d decodes) ratio=%.2f\n", + unbound_us, bound_us, N, bound_us / unbound_us); + // Pinning removes per-call guard work; it must not be materially slower. + EXPECT_LT(bound_us, unbound_us * 1.5); + }); } TEST(PinningBenchmark, DecodeBatchRestoresExactPriorMempolicy) { @@ -290,22 +330,24 @@ TEST(PinningInvariants, GuardSkipsMempolicyWhenCaptureFails) { TEST(PinningInvariants, BoundEntryPointsIssueNoPlacementSyscalls) { if (!shimCountersPresent()) GTEST_SKIP() << "affinity-counter shim not preloaded"; - for (const auto &ep : kCallerGuardedEntryPoints) { - SCOPED_TRACE(ep.name); - auto d = makeEntryReadyPinnedLut(); - d->bind_current_thread(); - auto before = thread_placement::capture(); - cudaqx_affinity_syscall_reset(); - ep.call(*d); - auto c = readShimCounts(); - EXPECT_EQ(c.setaff, 0) << "bound path issued sched_setaffinity"; - EXPECT_EQ(c.getaff, 0) << "bound path issued sched_getaffinity"; - EXPECT_EQ(c.setmem, 0) << "bound path issued set_mempolicy"; - EXPECT_EQ(c.getmem, 0) << "bound path issued get_mempolicy"; - auto after = thread_placement::capture(); - EXPECT_EQ(before, after) << "bound path changed placement: " - << before.describe_difference(after); - } + run_on_worker([] { + for (const auto &ep : kCallerGuardedEntryPoints) { + SCOPED_TRACE(ep.name); + auto d = makeEntryReadyPinnedLut(); + d->bind_current_thread(); + auto before = thread_placement::capture(); + cudaqx_affinity_syscall_reset(); + ep.call(*d); + auto c = readShimCounts(); + EXPECT_EQ(c.setaff, 0) << "bound path issued sched_setaffinity"; + EXPECT_EQ(c.getaff, 0) << "bound path issued sched_getaffinity"; + EXPECT_EQ(c.setmem, 0) << "bound path issued set_mempolicy"; + EXPECT_EQ(c.getmem, 0) << "bound path issued get_mempolicy"; + auto after = thread_placement::capture(); + EXPECT_EQ(before, after) << "bound path changed placement: " + << before.describe_difference(after); + } + }); } // decode_async decodes on a fresh std::async worker which can never be the @@ -315,18 +357,24 @@ TEST(PinningInvariants, BoundEntryPointsIssueNoPlacementSyscalls) { TEST(PinningInvariants, BoundDecodeAsyncGuardsWorkerAndLeavesCallerPlacement) { if (!shimCountersPresent()) GTEST_SKIP() << "affinity-counter shim not preloaded"; - auto d = makeEntryReadyPinnedLut(); - d->bind_current_thread(); - auto before = thread_placement::capture(); - cudaqx_affinity_syscall_reset(); - callDecodeAsync(*d); - EXPECT_GT(readShimCounts().setmem, 0) - << "decode_async's worker-thread guard must still attempt set_mempolicy " - "(the caller's binding must not leak to the worker)"; - auto after = thread_placement::capture(); - EXPECT_EQ(before, after) << "decode_async changed the CALLING thread's " - "placement: " - << before.describe_difference(after); + if (!get_mempolicy_usable()) + GTEST_SKIP() << "get_mempolicy blocked (container seccomp?); the guard " + "correctly skips set_mempolicy then, so setmem > 0 " + "cannot hold"; + run_on_worker([] { + auto d = makeEntryReadyPinnedLut(); + d->bind_current_thread(); + auto before = thread_placement::capture(); + cudaqx_affinity_syscall_reset(); + callDecodeAsync(*d); + EXPECT_GT(readShimCounts().setmem, 0) + << "decode_async's worker-thread guard must still attempt " + "set_mempolicy (the caller's binding must not leak to the worker)"; + auto after = thread_placement::capture(); + EXPECT_EQ(before, after) << "decode_async changed the CALLING thread's " + "placement: " + << before.describe_difference(after); + }); } // Unbound-path invariant, per entry point (incl. decode_async): the guard must @@ -335,6 +383,10 @@ TEST(PinningInvariants, BoundDecodeAsyncGuardsWorkerAndLeavesCallerPlacement) { TEST(PinningInvariants, UnboundEntryPointsApplyAndFullyRestoreGuard) { if (!shimCountersPresent()) GTEST_SKIP() << "affinity-counter shim not preloaded"; + if (!get_mempolicy_usable()) + GTEST_SKIP() << "get_mempolicy blocked (container seccomp?); the guard " + "correctly skips set_mempolicy then, so setmem > 0 " + "cannot hold"; std::vector entries(std::begin(kCallerGuardedEntryPoints), std::end(kCallerGuardedEntryPoints)); entries.push_back({"decode_async", callDecodeAsync}); @@ -351,3 +403,23 @@ TEST(PinningInvariants, UnboundEntryPointsApplyAndFullyRestoreGuard) { << before.describe_difference(after); } } + +// unbind_thread() must forget the registration so guards re-engage — the +// session calls it at teardown before the bound thread's id can be recycled. +TEST(PinningBenchmark, UnbindThreadRestoresGuarding) { + if (!cudaqx_affinity_syscall_count) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + run_on_worker([] { + auto d = makePinnedLut(); + d->bind_current_thread(); + cudaqx_affinity_syscall_reset(); + d->decode_batch(kChunk); + EXPECT_EQ(cudaqx_affinity_syscall_count(), 0) + << "bound thread must skip the guard"; + d->unbind_thread(); + cudaqx_affinity_syscall_reset(); + d->decode_batch(kChunk); + EXPECT_GT(cudaqx_affinity_syscall_count(), 0) + << "after unbind_thread() the guard must fire again"; + }); +} From 5ea8f0f16a9820ef50c8fc12d78541f763bd7a8b Mon Sep 17 00:00:00 2001 From: kvmto Date: Mon, 6 Jul 2026 17:04:55 +0000 Subject: [PATCH 11/16] test(qec): negative-path and perf-budget coverage for hardware pinning Fault-injection tests for every placement syscall (blocked get/set mempolicy, sched affinity, mbind, and a fully-blocked container mode): decodes stay correct, degradation is loud, restores hold. Knob-contract negatives (typo'd key, wrong types naming the key, nonexistent NUMA node) and a silence contract for knobless users. Binding lifecycle negatives (rebind migration, cross-thread unbind, bind races). First runtime coverage of the session conflict gate (cpu_affinity/device/numa disagreement -> warn + guards stay active, corrections bit-correct). Python negatives for both get_decoder overloads. Perf-regression budgets: 7 placement syscalls and 1 sysfs read per unbound decode, enforced via the extended LD_PRELOAD shim (per-syscall fault injection, mbind and sysfs-open counters). Signed-off-by: kvmto --- libs/qec/python/tests/test_decoder.py | 86 +++++ .../pymatching/test_pymatching_realtime.cpp | 229 +++++++++++++ .../support/affinity_syscall_shim.cpp | 174 +++++++++- libs/qec/unittests/test_decoders.cpp | 64 ++++ libs/qec/unittests/test_device_affinity.cpp | 14 + libs/qec/unittests/test_pinning_benchmark.cpp | 319 ++++++++++++++++++ 6 files changed, 881 insertions(+), 5 deletions(-) diff --git a/libs/qec/python/tests/test_decoder.py b/libs/qec/python/tests/test_decoder.py index ac991f07a..29eb97793 100644 --- a/libs/qec/python/tests/test_decoder.py +++ b/libs/qec/python/tests/test_decoder.py @@ -1074,5 +1074,91 @@ def decode(self, syndrome): assert "numa_node_id" not in received +def test_native_decoder_accepts_affinity_kwargs_from_python(): + # Native (C++-registered) decoders route through decoder::get(), which + # strips the affinity keys before the plugin constructor and stores them + # on the base via set_hardware_params(). + H = np.array([[1, 1, 0], [0, 1, 1]], dtype=np.uint8) + d = qec.get_decoder("multi_error_lut", H, numa_node_id=0) + assert d.numa_node_id() == 0 + assert d.cuda_device_id() == -1 + r = d.decode([0.0, 0.0]) + assert r is not None + + +def test_dem_text_overload_strips_affinity_kwargs_for_python_decoder(): + + @qec.decoder("dem_strict_affinity_byod_neg") + class DemStrictAffinityDecoder: + # Strict signature, deliberately NO **kwargs: the DEM-text path + # injects O and error_rate_vec kwargs, so exactly those (plus H) + # must arrive here — the affinity keys must already be stripped. + def __init__(self, H, O, error_rate_vec): + qec.Decoder.__init__(self, H) + self.num_obs = O.shape[0] + + def decode(self, syndrome): + r = qec.DecoderResult() + r.converged = True + r.result = [0.0] * self.num_obs + return r + + dem_text = ("error(0.1) D0 L0\n" + "error(0.1) D1 L0\n" + "error(0.05) D0 D1\n") + d = qec.get_decoder("dem_strict_affinity_byod_neg", + dem_text, + numa_node_id=0) + assert d.numa_node_id() == 0 + r = d.decode([0.0, 0.0]) + assert r.converged is True + + +def test_uninitialized_base_with_affinity_kwargs_raises_actionable_error(): + + @qec.decoder("no_base_byod_neg") + class NoBaseDecoder: + + def __init__(self, H): + pass # deliberately never calls qec.Decoder.__init__ + + def decode(self, syndrome): + return None + + H = np.array([[1, 1, 0], [0, 1, 1]], dtype=np.uint8) + with pytest.raises(RuntimeError, match="qec.Decoder.__init__"): + qec.get_decoder("no_base_byod_neg", H, numa_node_id=0) + + +def test_explicit_negative_cuda_device_id_raises(): + H = np.array([[1, 1, 0], [0, 1, 1]], dtype=np.uint8) + # The failure comes from the kwargs converter: hetMapFromKwargs stores + # Python ints as std::size_t, and nanobind's cast rejects negative ints + # for unsigned storage. That nb::cast_error surfaces as TypeError or + # RuntimeError depending on the nanobind translation, with no stable + # message text to match. + with pytest.raises((RuntimeError, TypeError)): + qec.get_decoder("multi_error_lut", H, cuda_device_id=-1) + + +def test_non_integral_cpu_affinity_raises_naming_integers(): + H = np.array([[1, 1, 0], [0, 1, 1]], dtype=np.uint8) + # Python lists arrive in the heterogeneous map as vector; + # read_cpu_affinity() rejects non-integral core ids at construction. + with pytest.raises(RuntimeError, match="must be integers"): + qec.get_decoder("multi_error_lut", + H, + numa_node_id=0, + cpu_affinity=[0.5]) + + +def test_invalid_mempolicy_string_raises_naming_the_knob(): + H = np.array([[1, 1, 0], [0, 1, 1]], dtype=np.uint8) + # read_mempolicy() accepts only "preferred" or "bind"; anything else is + # malformed explicit input and must throw, naming the knob. + with pytest.raises(RuntimeError, match="mempolicy"): + qec.get_decoder("multi_error_lut", H, numa_node_id=0, mempolicy="bnid") + + if __name__ == "__main__": pytest.main() diff --git a/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp b/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp index 34bfd250d..16b34f3eb 100644 --- a/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp +++ b/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp @@ -14,9 +14,14 @@ #include +#include +#include + #include +#include #include #include +#include #include #include @@ -226,3 +231,227 @@ TEST(PyMatchingRealtime, ConfiguresViaRealtimeDecoderConfig) { config::finalize_decoders(); } + +//============================================================================== +// Session-level hardware-pinning conflict tests (consensus gate in +// qec_realtime_session::initialize()). Contract under test: decoders that +// disagree on an affinity knob must degrade LOUDLY (a CUDA_QEC_WARN naming the +// knob) but never break the session -- initialize() succeeds, every decoder +// still decodes correctly through the per-call guards, finalize() is clean. +//============================================================================== + +namespace { + +// H (3x2) shared by the conflict tests; syndrome {1,1,0} decodes to {1,0} +// (the same known-good pymatching mapping CheckRegularEdges relies on). +const std::vector kConflictH = {1, 0, 1, 1, 0, 1}; +constexpr std::size_t kConflictSyndromeSize = 3; +constexpr std::size_t kConflictBlockSize = 2; + +// Two identical pymatching decoders with ids {0, 1}. Affinity knobs are +// applied by each test via the public decoder::set_hardware_params, so the +// knobless (B4) path never touches the knob API at all. +DecoderVec make_session_pair() { + auto a = make_pymatching_decoders(kConflictH, kConflictSyndromeSize, + kConflictBlockSize); + auto b = make_pymatching_decoders(kConflictH, kConflictSyndromeSize, + kConflictBlockSize); + b[0]->set_decoder_id(1); + DecoderVec decoders; + decoders.push_back(std::move(a[0])); + decoders.push_back(std::move(b[0])); + return decoders; +} + +// HOST-mode sessions are process-global-exclusive (g_active_decoders), so the +// slot must be released on EVERY exit path, assertion failures included. +// finalize() is documented idempotent + destructor-safe, so an extra explicit +// finalize() in the happy path is harmless. +struct session_finalizer { + cudaq::qec::realtime::qec_realtime_session &session; + ~session_finalizer() { session.finalize(); } +}; + +// initialize() while capturing stderr. Never lets an exception escape while +// the gtest capture is active (a dangling capture would poison later tests); +// the caller asserts on `threw` after the capture is closed. +std::string +initialize_capturing_stderr(cudaq::qec::realtime::qec_realtime_session &session, + bool &threw, std::string &what) { + threw = false; + what.clear(); + testing::internal::CaptureStderr(); + try { + session.initialize(); + } catch (const std::exception &e) { + threw = true; + what = e.what(); + } catch (...) { + threw = true; + what = "non-std::exception thrown"; + } + return testing::internal::GetCapturedStderr(); +} + +// Round-trip one known syndrome through `decoder_id` and require the exact +// pymatching correction for kConflictH. +// CUDA device count for the B2 gate. This test target does not link cudart +// directly (a direct cudaGetDeviceCount reference fails to link with "DSO +// missing from command line") and CMake edits are out of scope, but +// libcudart.so is already in the process image as a dependency of +// libcudaq-qec -- so resolve the symbol dynamically for the probe only. +// Returns -1 when the symbol or the CUDA runtime is unavailable. +int probe_cuda_device_count() { + void *sym = dlsym(RTLD_DEFAULT, "cudaGetDeviceCount"); + if (!sym) + return -1; + // ABI: cudaError_t is an int-sized enum; cudaSuccess == 0. + auto get_count = reinterpret_cast(sym); + int count = 0; + if (get_count(&count) != 0) + return -1; + return count; +} + +void expect_decoder_still_decodes( + cudaq::qec::realtime::qec_realtime_session &session, std::size_t decoder_id, + std::uint64_t tag) { + const std::vector syndrome{1, 1, 0}; + cudaq::qec::decoding::rpc_producer::enqueue_syndromes( + session, decoder_id, syndrome.data(), syndrome.size(), tag); + std::vector corrections(kConflictBlockSize, 0xCC); + cudaq::qec::decoding::rpc_producer::get_corrections( + session, decoder_id, corrections.data(), corrections.size(), + /*reset=*/1); + EXPECT_EQ(corrections, (std::vector{1, 0})) + << "decoder " << decoder_id + << " must still decode correctly after a session-level pinning conflict"; +} + +} // namespace + +// Conflict gate: same NUMA node, different non-empty cpu_affinity lists. +TEST(PyMatchingRealtime, SessionCpuAffinityConflictWarnsAndStillDecodes) { +#if defined(__linux__) + cpu_set_t allowed; + CPU_ZERO(&allowed); + ASSERT_EQ(sched_getaffinity(0, sizeof(allowed), &allowed), 0); + if (!CPU_ISSET(0, &allowed) || !CPU_ISSET(1, &allowed)) + GTEST_SKIP() << "CPUs 0 and 1 are not both in the allowed cpuset"; + + auto decoders = make_session_pair(); + cudaqx::heterogeneous_map knobs_a; + knobs_a.insert("numa_node_id", 0); // node 0 always exists + knobs_a.insert("cpu_affinity", std::vector{0}); + decoders[0]->set_hardware_params(knobs_a); + cudaqx::heterogeneous_map knobs_b; + knobs_b.insert("numa_node_id", 0); + knobs_b.insert("cpu_affinity", std::vector{1}); + decoders[1]->set_hardware_params(knobs_b); + + cudaq::qec::realtime::qec_realtime_session session(decoders); + session_finalizer fin{session}; + bool threw = false; + std::string what; + const std::string err = initialize_capturing_stderr(session, threw, what); + ASSERT_FALSE(threw) << "conflicting cpu_affinity lists must degrade to " + "per-call guards, not throw: " + << what; + EXPECT_NE(err.find("different cpu_affinity lists"), std::string::npos) + << "conflict must be loud; captured stderr: [" << err << "]"; + + expect_decoder_still_decodes(session, /*decoder_id=*/0, /*tag=*/1); + expect_decoder_still_decodes(session, /*decoder_id=*/1, /*tag=*/2); + EXPECT_NO_THROW(session.finalize()); +#else + GTEST_SKIP() << "Linux-only (sched_getaffinity probe)"; +#endif +} + +// Conflict gate: different cuda_device_ids (no numa knob -- exercises the +// dev-conflict leg; numa_node_id may be soft-derived from the device but on a +// conflict-free node set that leg stays quiet). +TEST(PyMatchingRealtime, SessionCudaDeviceConflictWarnsAndStillDecodes) { + const int device_count = probe_cuda_device_count(); + if (device_count < 2) + GTEST_SKIP() << "needs >= 2 CUDA devices, have " << device_count; + + auto decoders = make_session_pair(); + cudaqx::heterogeneous_map knobs_a; + knobs_a.insert("cuda_device_id", 0); + decoders[0]->set_hardware_params(knobs_a); + cudaqx::heterogeneous_map knobs_b; + knobs_b.insert("cuda_device_id", 1); + decoders[1]->set_hardware_params(knobs_b); + + cudaq::qec::realtime::qec_realtime_session session(decoders); + session_finalizer fin{session}; + bool threw = false; + std::string what; + const std::string err = initialize_capturing_stderr(session, threw, what); + ASSERT_FALSE(threw) << "conflicting cuda_device_ids must degrade to " + "per-call guards, not throw: " + << what; + EXPECT_NE(err.find("different cuda_device_ids"), std::string::npos) + << "conflict must be loud; captured stderr: [" << err << "]"; + + expect_decoder_still_decodes(session, /*decoder_id=*/0, /*tag=*/1); + expect_decoder_still_decodes(session, /*decoder_id=*/1, /*tag=*/2); + EXPECT_NO_THROW(session.finalize()); +} + +// Conflict gate: different numa_node_ids. Requires a multi-node host (CI +// lane); must SKIP cleanly on single-node boxes. +TEST(PyMatchingRealtime, SessionNumaNodeConflictWarnsAndStillDecodes) { + std::ifstream node1("/sys/devices/system/node/node1/cpulist"); + if (!node1.is_open()) + GTEST_SKIP() << "single NUMA node host; numa conflict not testable"; + + auto decoders = make_session_pair(); + cudaqx::heterogeneous_map knobs_a; + knobs_a.insert("numa_node_id", 0); + decoders[0]->set_hardware_params(knobs_a); + cudaqx::heterogeneous_map knobs_b; + knobs_b.insert("numa_node_id", 1); + decoders[1]->set_hardware_params(knobs_b); + + cudaq::qec::realtime::qec_realtime_session session(decoders); + session_finalizer fin{session}; + bool threw = false; + std::string what; + const std::string err = initialize_capturing_stderr(session, threw, what); + ASSERT_FALSE(threw) << "conflicting numa_node_ids must disable session NUMA " + "pinning, not throw: " + << what; + EXPECT_NE(err.find("different numa_node_ids"), std::string::npos) + << "conflict must be loud; captured stderr: [" << err << "]"; + + expect_decoder_still_decodes(session, /*decoder_id=*/0, /*tag=*/1); + expect_decoder_still_decodes(session, /*decoder_id=*/1, /*tag=*/2); + EXPECT_NO_THROW(session.finalize()); +} + +// Silence contract: a session over decoders that never +// touched a pinning knob must emit ZERO affinity noise ("[cudaq-qec +// affinity]" is the fprintf marker of hardware_affinity.h's affinity_warn) +// across construct + initialize + finalize. +TEST(PyMatchingRealtime, SessionKnoblessDecodersEmitNoAffinityNoise) { + auto decoders = make_session_pair(); // set_hardware_params never called + bool threw = false; + std::string what; + testing::internal::CaptureStderr(); + { + cudaq::qec::realtime::qec_realtime_session session(decoders); + session_finalizer fin{session}; + try { + session.initialize(); + } catch (const std::exception &e) { + threw = true; + what = e.what(); + } + } // finalize() runs here, inside the capture + const std::string err = testing::internal::GetCapturedStderr(); + ASSERT_FALSE(threw) << "knobless session must initialize cleanly: " << what; + EXPECT_EQ(err.find("[cudaq-qec affinity]"), std::string::npos) + << "knobless session must be affinity-silent, got: [" << err << "]"; +} diff --git a/libs/qec/unittests/support/affinity_syscall_shim.cpp b/libs/qec/unittests/support/affinity_syscall_shim.cpp index 926c5c564..3c6cdeb0b 100644 --- a/libs/qec/unittests/support/affinity_syscall_shim.cpp +++ b/libs/qec/unittests/support/affinity_syscall_shim.cpp @@ -7,17 +7,32 @@ ******************************************************************************/ // Test-only LD_PRELOAD interposer: counts the placement-related syscalls so a // test can assert a bound decode loop issues none (and an unbound one does). -// Counts four syscalls: sched_setaffinity / sched_getaffinity (glibc symbols) -// and SYS_set_mempolicy / SYS_get_mempolicy (issued through glibc's syscall(2) -// wrapper, so that wrapper is interposed too). Not linked into the library. +// Counts five syscalls: sched_setaffinity / sched_getaffinity (glibc symbols) +// and SYS_set_mempolicy / SYS_get_mempolicy / SYS_mbind (issued through +// glibc's syscall(2) wrapper, so that wrapper is interposed too). Also counts +// opens of /sys/devices/system/node/* (the per-guard cpulist read) via the +// openat/openat64 glibc symbols. Not linked into the library. +// +// Fault injection (getenv per call, so a test can toggle around one decode): +// CUDAQX_SHIM_FAIL_SETAFFINITY sched_setaffinity -> EPERM, -1 +// CUDAQX_SHIM_FAIL_GETAFFINITY sched_getaffinity -> EPERM, -1 +// CUDAQX_SHIM_FAIL_SET_MEMPOLICY SYS_set_mempolicy -> EPERM, -1 +// CUDAQX_SHIM_FAIL_GET_MEMPOLICY SYS_get_mempolicy -> EPERM, -1 +// CUDAQX_SHIM_FAIL_MBIND SYS_mbind -> EPERM, -1 +// CUDAQX_SHIM_FAIL_ALL_PLACEMENT all of the above -> EPERM, -1 +// Every counter counts ATTEMPTS: the count is incremented before injection. +// The sysfs open counters have NO fault injection. #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include #include #include +#include #include +#include #include +#include #include #include #include @@ -27,6 +42,16 @@ std::atomic g_setaff{0}; std::atomic g_getaff{0}; std::atomic g_setmem{0}; std::atomic g_getmem{0}; +std::atomic g_mbind{0}; +std::atomic g_node_sysfs_open{0}; + +// True when the call must be failed: either its specific injection variable or +// the blanket CUDAQX_SHIM_FAIL_ALL_PLACEMENT is set. getenv per call so a test +// can set/unset around a single decode. +bool fail_injected(const char *specific) { + return std::getenv(specific) != nullptr || + std::getenv("CUDAQX_SHIM_FAIL_ALL_PLACEMENT") != nullptr; +} // dlsym may itself invoke interposed functions; the thread-local flag breaks // that recursion (the inner call sees "resolving" and reports ENOSYS instead @@ -47,17 +72,38 @@ syscall_fn real_syscall_lazy() { cached.store(fn, std::memory_order_release); return fn; } + +// openat's optional 4th argument exists only for these flags. +bool open_needs_mode(int flags) { + if (flags & O_CREAT) + return true; +#ifdef O_TMPFILE + if ((flags & O_TMPFILE) == O_TMPFILE) + return true; +#endif + return false; +} + +void count_node_sysfs_path(const char *pathname) { + if (pathname && std::strstr(pathname, "/sys/devices/system/node/")) + g_node_sysfs_open.fetch_add(1, std::memory_order_relaxed); +} } // namespace extern "C" { // Interpose the glibc symbols; forward to the real ones via RTLD_NEXT. +// Counts first (attempts), then applies fault injection. int sched_setaffinity(pid_t pid, size_t cpusetsize, const cpu_set_t *mask) { static int (*real)(pid_t, size_t, const cpu_set_t *) = nullptr; if (!real) real = reinterpret_cast( dlsym(RTLD_NEXT, "sched_setaffinity")); g_setaff.fetch_add(1, std::memory_order_relaxed); + if (fail_injected("CUDAQX_SHIM_FAIL_SETAFFINITY")) { + errno = EPERM; + return -1; + } return real(pid, cpusetsize, mask); } @@ -67,6 +113,10 @@ int sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t *mask) { real = reinterpret_cast( dlsym(RTLD_NEXT, "sched_getaffinity")); g_getaff.fetch_add(1, std::memory_order_relaxed); + if (fail_injected("CUDAQX_SHIM_FAIL_GETAFFINITY")) { + errno = EPERM; + return -1; + } return real(pid, cpusetsize, mask); } @@ -81,22 +131,40 @@ long syscall(long number, ...) { switch (number) { case SYS_set_mempolicy: g_setmem.fetch_add(1, std::memory_order_relaxed); + if (fail_injected("CUDAQX_SHIM_FAIL_SET_MEMPOLICY")) { + errno = EPERM; + return -1; + } break; case SYS_get_mempolicy: g_getmem.fetch_add(1, std::memory_order_relaxed); // Fault injection: simulate a seccomp profile that blocks get_mempolicy // while allowing set_mempolicy (the un-restorable-policy scenario). - // getenv per call so a test can toggle it around a single decode. - if (std::getenv("CUDAQX_SHIM_FAIL_GET_MEMPOLICY")) { + if (fail_injected("CUDAQX_SHIM_FAIL_GET_MEMPOLICY")) { + errno = EPERM; + return -1; + } + break; + case SYS_mbind: + g_mbind.fetch_add(1, std::memory_order_relaxed); + if (fail_injected("CUDAQX_SHIM_FAIL_MBIND")) { errno = EPERM; return -1; } break; case SYS_sched_setaffinity: g_setaff.fetch_add(1, std::memory_order_relaxed); + if (fail_injected("CUDAQX_SHIM_FAIL_SETAFFINITY")) { + errno = EPERM; + return -1; + } break; case SYS_sched_getaffinity: g_getaff.fetch_add(1, std::memory_order_relaxed); + if (fail_injected("CUDAQX_SHIM_FAIL_GETAFFINITY")) { + errno = EPERM; + return -1; + } break; } syscall_fn real = real_syscall_lazy(); @@ -107,6 +175,96 @@ long syscall(long number, ...) { return real(number, a0, a1, a2, a3, a4, a5); } +// Count opens of the node-topology sysfs tree (build_node_cpuset's per-guard +// /sys/devices/system/node/node/cpulist read). On this platform libstdc++'s +// ifstream is stdio-based (__basic_file wraps FILE*), so the file is opened +// through glibc's fopen -- whose INTERNAL open bypasses every interposable +// open/openat symbol (measured: openat/openat64 alone counted 0, adding +// open/open64 still counted 0). fopen/fopen64 are therefore interposed too; +// the open* spellings are kept so the counter stays robust across libstdc++ +// builds that call them directly. Pass-through only: NO fault injection. +FILE *fopen(const char *pathname, const char *mode) { + static FILE *(*real)(const char *, const char *) = nullptr; + if (!real) + real = reinterpret_cast( + dlsym(RTLD_NEXT, "fopen")); + count_node_sysfs_path(pathname); + return real(pathname, mode); +} + +FILE *fopen64(const char *pathname, const char *mode) { + static FILE *(*real)(const char *, const char *) = nullptr; + if (!real) + real = reinterpret_cast( + dlsym(RTLD_NEXT, "fopen64")); + count_node_sysfs_path(pathname); + return real(pathname, mode); +} + +int open(const char *pathname, int flags, ...) { + static int (*real)(const char *, int, ...) = nullptr; + if (!real) + real = reinterpret_cast( + dlsym(RTLD_NEXT, "open")); + count_node_sysfs_path(pathname); + if (open_needs_mode(flags)) { + va_list ap; + va_start(ap, flags); + mode_t mode = va_arg(ap, mode_t); + va_end(ap); + return real(pathname, flags, mode); + } + return real(pathname, flags); +} + +int open64(const char *pathname, int flags, ...) { + static int (*real)(const char *, int, ...) = nullptr; + if (!real) + real = reinterpret_cast( + dlsym(RTLD_NEXT, "open64")); + count_node_sysfs_path(pathname); + if (open_needs_mode(flags)) { + va_list ap; + va_start(ap, flags); + mode_t mode = va_arg(ap, mode_t); + va_end(ap); + return real(pathname, flags, mode); + } + return real(pathname, flags); +} + +int openat(int dirfd, const char *pathname, int flags, ...) { + static int (*real)(int, const char *, int, ...) = nullptr; + if (!real) + real = reinterpret_cast( + dlsym(RTLD_NEXT, "openat")); + count_node_sysfs_path(pathname); + if (open_needs_mode(flags)) { + va_list ap; + va_start(ap, flags); + mode_t mode = va_arg(ap, mode_t); + va_end(ap); + return real(dirfd, pathname, flags, mode); + } + return real(dirfd, pathname, flags); +} + +int openat64(int dirfd, const char *pathname, int flags, ...) { + static int (*real)(int, const char *, int, ...) = nullptr; + if (!real) + real = reinterpret_cast( + dlsym(RTLD_NEXT, "openat64")); + count_node_sysfs_path(pathname); + if (open_needs_mode(flags)) { + va_list ap; + va_start(ap, flags); + mode_t mode = va_arg(ap, mode_t); + va_end(ap); + return real(dirfd, pathname, flags, mode); + } + return real(dirfd, pathname, flags); +} + // Read/reset hooks the test resolves (weakly) from the preloaded shim. // Legacy pair kept for existing tests: count == sched_setaffinity count. long cudaqx_affinity_syscall_count() { @@ -117,6 +275,8 @@ void cudaqx_affinity_syscall_reset() { g_getaff.store(0, std::memory_order_relaxed); g_setmem.store(0, std::memory_order_relaxed); g_getmem.store(0, std::memory_order_relaxed); + g_mbind.store(0, std::memory_order_relaxed); + g_node_sysfs_open.store(0, std::memory_order_relaxed); } // Per-syscall counters. long cudaqx_sched_setaffinity_count() { @@ -131,4 +291,8 @@ long cudaqx_set_mempolicy_count() { long cudaqx_get_mempolicy_count() { return g_getmem.load(std::memory_order_relaxed); } +long cudaqx_mbind_count() { return g_mbind.load(std::memory_order_relaxed); } +long cudaqx_node_sysfs_open_count() { + return g_node_sysfs_open.load(std::memory_order_relaxed); +} } diff --git a/libs/qec/unittests/test_decoders.cpp b/libs/qec/unittests/test_decoders.cpp index d8d479528..442ad2108 100644 --- a/libs/qec/unittests/test_decoders.cpp +++ b/libs/qec/unittests/test_decoders.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1665,3 +1666,66 @@ TEST(DecoderAffinity, PinnedThreadDecodePreservesCallersBinding) { GTEST_SKIP() << "Linux-only"; #endif } + +// ---- Knob-contract negative tests -------------------------------------- + +// T1.1: a typo'd knob key must not accidentally pin anything -- the tolerant +// plugin ignores unknown keys, so both affinity knobs stay unset and decoding +// still works. +TEST(HardwarePinningNegative, TypoedKnobKeyIsSilentlyIgnoredByTolerantPlugin) { + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + params.insert("cuda_device", 1); // typo: should be cuda_device_id + auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); + EXPECT_EQ(d->cuda_device_id(), -1) << "typo must not accidentally pin"; + EXPECT_EQ(d->numa_node_id(), -1); + std::vector> batch = {{0.1f, 0.1f}}; + EXPECT_NO_THROW((void)d->decode_batch(batch)); +} + +// T1.7: silence contract -- users who never touch the pinning knobs must never +// see an affinity warning. +TEST(HardwarePinningNegative, UnpinnedUsersSeeZeroAffinityWarnings) { + cudaqx::tensor H({2, 3}); + testing::internal::CaptureStderr(); + auto d = cudaq::qec::decoder::get("multi_error_lut", H, + cudaqx::heterogeneous_map{}); + std::vector> batch = {{0.1f, 0.1f}}; + (void)d->decode_batch(batch); + std::string err = testing::internal::GetCapturedStderr(); + EXPECT_EQ(err.find("[cudaq-qec affinity]"), std::string::npos) + << "knobless usage must be warning-silent, got: " << err; +} + +// T2.12: a numa_node_id that is encodable (< 64) but does not exist on this +// host must degrade loudly (warning naming the node) and still decode +// correctly, unpinned. +#if defined(__linux__) +TEST(HardwarePinningNegative, NonexistentNodeWarnsAndRunsUnpinned) { + // Pick a node id in [0,64) that does not exist on this host. + int missing = -1; + for (int n = 8; n < 64; ++n) { + std::ifstream f("/sys/devices/system/node/node" + std::to_string(n) + + "/cpulist"); + if (!f.is_open()) { + missing = n; + break; + } + } + if (missing < 0) + GTEST_SKIP() << "every node id in [8,64) exists on this host"; + cudaqx::tensor H({2, 3}); + cudaqx::heterogeneous_map params; + params.insert("numa_node_id", missing); + testing::internal::CaptureStderr(); + auto d = cudaq::qec::decoder::get("multi_error_lut", H, params); + std::vector> batch = {{0.1f, 0.1f}}; + auto results = d->decode_batch(batch); + std::string err = testing::internal::GetCapturedStderr(); + ASSERT_EQ(results.size(), batch.size()); + EXPECT_TRUE(results[0].converged) << "must degrade, not corrupt"; + EXPECT_NE(err.find("numa_node_id " + std::to_string(missing)), + std::string::npos) + << "degradation must be loud"; +} +#endif diff --git a/libs/qec/unittests/test_device_affinity.cpp b/libs/qec/unittests/test_device_affinity.cpp index e0916ab09..2e2208812 100644 --- a/libs/qec/unittests/test_device_affinity.cpp +++ b/libs/qec/unittests/test_device_affinity.cpp @@ -195,3 +195,17 @@ TEST(HardwareAffinity, BindRegionNode64Throws) { std::free(p); } #endif + +// T1.2: a correctly named knob with the WRONG value type must throw, and the +// error must name the offending key so the user can find it. +TEST(DeviceAffinity, WrongTypedKnobThrowsNamingTheKey) { + heterogeneous_map m; + m.insert("cuda_device_id", std::string("0")); + try { + (void)read_cuda_device_id(m); + FAIL() << "expected a throw for string-typed cuda_device_id"; + } catch (const std::exception &e) { + EXPECT_NE(std::string(e.what()).find("cuda_device_id"), std::string::npos) + << "error must name the key; got: " << e.what(); + } +} diff --git a/libs/qec/unittests/test_pinning_benchmark.cpp b/libs/qec/unittests/test_pinning_benchmark.cpp index 5f3c1f1d7..5eaa1c286 100644 --- a/libs/qec/unittests/test_pinning_benchmark.cpp +++ b/libs/qec/unittests/test_pinning_benchmark.cpp @@ -7,10 +7,13 @@ ******************************************************************************/ #include "support/thread_placement.h" #include "cudaq/qec/decoder.h" +#include #include #include #include +#include #include +#include #include #include #if defined(__linux__) @@ -28,6 +31,8 @@ extern "C" __attribute__((weak)) long cudaqx_sched_setaffinity_count(); extern "C" __attribute__((weak)) long cudaqx_sched_getaffinity_count(); extern "C" __attribute__((weak)) long cudaqx_set_mempolicy_count(); extern "C" __attribute__((weak)) long cudaqx_get_mempolicy_count(); +extern "C" __attribute__((weak)) long cudaqx_mbind_count(); +extern "C" __attribute__((weak)) long cudaqx_node_sysfs_open_count(); namespace { cudaqx::tensor makeH() { @@ -423,3 +428,317 @@ TEST(PinningBenchmark, UnbindThreadRestoresGuarding) { << "after unbind_thread() the guard must fire again"; }); } + +// ---- Fault-injection negative tests ------------------------------------ +// Each test sets AND unsets its own CUDAQX_SHIM_FAIL_* variable so no +// injection state leaks across tests. + +// T1.4: a container that blocks EVERY placement syscall (blanket injection). +// The decode must still be correct, and the degradation must be loud. +TEST(PinningNegative, FullyBlockedSyscallsDegradeLoudlyNotWrongly) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + run_on_worker([] { + setenv("CUDAQX_SHIM_FAIL_ALL_PLACEMENT", "1", 1); + auto d = makeEntryReadyPinnedLut(); + testing::internal::CaptureStderr(); + auto results = d->decode_batch(kChunk); + std::string err = testing::internal::GetCapturedStderr(); + unsetenv("CUDAQX_SHIM_FAIL_ALL_PLACEMENT"); + ASSERT_EQ(results.size(), kChunk.size()); + for (auto &r : results) + EXPECT_TRUE(r.converged) << "degraded env must not corrupt decode"; + EXPECT_NE(err.find("[cudaq-qec affinity] WARNING"), std::string::npos) + << "degradation must be loud"; + }); +} + +// T2.8: when the prior affinity cannot be read, the guard could never restore +// it, so it must not call sched_setaffinity at all (only-if-restorable rule). +TEST(PinningInvariants, BlockedGetaffinityAppliesNoAffinityAtAll) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + run_on_worker([] { + auto d = makePinnedLut(); + auto before = thread_placement::capture(); + setenv("CUDAQX_SHIM_FAIL_GETAFFINITY", "1", 1); + cudaqx_affinity_syscall_reset(); + auto results = d->decode_batch(kChunk); + unsetenv("CUDAQX_SHIM_FAIL_GETAFFINITY"); + EXPECT_EQ(readShimCounts().setaff, 0) + << "unreadable prior affinity must mean NO sched_setaffinity"; + auto after = thread_placement::capture(); + EXPECT_EQ(before, after); + ASSERT_EQ(results.size(), kChunk.size()); + EXPECT_TRUE(results[0].converged); + }); +} + +// T2.9: sched_setaffinity blocked (locked cpuset). The independent mempolicy +// half must still run, the failure must be loud, and nothing may leak. +TEST(PinningInvariants, BlockedSetaffinityStillAppliesMempolicyHalf) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + if (!get_mempolicy_usable()) + GTEST_SKIP() << "get_mempolicy blocked (container seccomp?); the guard " + "correctly skips set_mempolicy then, so setmem > 0 " + "cannot hold"; + run_on_worker([] { + auto d = makePinnedLut(); + auto before = thread_placement::capture(); + setenv("CUDAQX_SHIM_FAIL_SETAFFINITY", "1", 1); + cudaqx_affinity_syscall_reset(); + testing::internal::CaptureStderr(); + auto results = d->decode_batch(kChunk); + std::string err = testing::internal::GetCapturedStderr(); + unsetenv("CUDAQX_SHIM_FAIL_SETAFFINITY"); + EXPECT_GT(readShimCounts().setmem, 0) + << "mempolicy half must still run when sched_setaffinity is blocked"; + auto after = thread_placement::capture(); + EXPECT_EQ(before, after) + << "blocked sched_setaffinity leaked a placement change: " + << before.describe_difference(after); + ASSERT_EQ(results.size(), kChunk.size()); + EXPECT_TRUE(results[0].converged); + EXPECT_NE(err.find("sched_setaffinity"), std::string::npos) + << "blocked sched_setaffinity must be loud; got: " << err; + }); +} + +// T2.10: set_mempolicy blocked (no CAP_SYS_NICE). The independent affinity +// half must still run, the failure must be loud, and nothing may leak. +TEST(PinningInvariants, BlockedSetMempolicyStillAppliesAffinityHalf) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + if (!get_mempolicy_usable()) + GTEST_SKIP() << "get_mempolicy blocked (container seccomp?); the guard " + "then skips the mempolicy half entirely and never prints " + "\"set_mempolicy failed\""; + run_on_worker([] { + auto d = makePinnedLut(); + auto before = thread_placement::capture(); + setenv("CUDAQX_SHIM_FAIL_SET_MEMPOLICY", "1", 1); + cudaqx_affinity_syscall_reset(); + testing::internal::CaptureStderr(); + auto results = d->decode_batch(kChunk); + std::string err = testing::internal::GetCapturedStderr(); + unsetenv("CUDAQX_SHIM_FAIL_SET_MEMPOLICY"); + EXPECT_GT(readShimCounts().setaff, 0) + << "affinity half must still run when set_mempolicy is blocked"; + auto after = thread_placement::capture(); + EXPECT_EQ(before, after) + << "blocked set_mempolicy leaked a placement change: " + << before.describe_difference(after); + ASSERT_EQ(results.size(), kChunk.size()); + EXPECT_TRUE(results[0].converged); + EXPECT_NE(err.find("set_mempolicy failed"), std::string::npos) + << "blocked set_mempolicy must be loud; got: " << err; + }); +} + +// ---- Binding lifecycle negative tests +// ----------------------------------------------- + +// T3.13: re-binding on a second thread migrates the guard-skip: the new owner +// decodes guard-free, the previous owner is guarded again. promise/future +// handshakes keep both threads alive through every measured decode (no thread +// id can be recycled) and serialize the decodes (the shim counters are global). +TEST(PinningBenchmark, RebindMigratesGuardSkipToNewOwner) { + if (!cudaqx_affinity_syscall_count) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + auto d = makePinnedLut(); + std::promise a_bound_p, b_done_p; + auto a_bound_f = a_bound_p.get_future(); + auto b_done_f = b_done_p.get_future(); + long a_first = -1, b_count = -1, a_second = -1; + std::thread ta([&] { + d->bind_current_thread(); + cudaqx_affinity_syscall_reset(); + d->decode_batch(kChunk); + a_first = cudaqx_affinity_syscall_count(); + a_bound_p.set_value(); + b_done_f.wait(); // B has re-bound and decoded + cudaqx_affinity_syscall_reset(); + d->decode_batch(kChunk); + a_second = cudaqx_affinity_syscall_count(); + }); + std::thread tb([&] { + a_bound_f.wait(); // A is the current owner + d->bind_current_thread(); // rebind: ownership migrates to B + cudaqx_affinity_syscall_reset(); + d->decode_batch(kChunk); + b_count = cudaqx_affinity_syscall_count(); + b_done_p.set_value(); + }); + ta.join(); + tb.join(); + EXPECT_EQ(a_first, 0) << "owner A must decode guard-free before the rebind"; + EXPECT_EQ(b_count, 0) << "new owner B must decode guard-free after rebinding"; + EXPECT_GT(a_second, 0) + << "previous owner A must be guarded again after B re-bound"; +} + +// T3.13b: binding twice on the same thread is idempotent -- no throw, and the +// thread still decodes guard-free. +TEST(PinningBenchmark, DoubleBindIsIdempotent) { + if (!cudaqx_affinity_syscall_count) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + run_on_worker([] { + auto d = makePinnedLut(); + d->bind_current_thread(); + EXPECT_NO_THROW(d->bind_current_thread()); + cudaqx_affinity_syscall_reset(); + d->decode_batch(kChunk); + EXPECT_EQ(cudaqx_affinity_syscall_count(), 0); + }); +} + +// T3.14: unbind_thread() called from a DIFFERENT thread clears the owner; the +// still-alive previously-bound thread must be guarded again. The worker stays +// alive across the unbind via a promise/future handshake (no sleeps). +TEST(PinningBenchmark, UnbindFromAnotherThreadRestoresGuarding) { + if (!cudaqx_affinity_syscall_count) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + auto d = makePinnedLut(); + std::promise bound_p, unbound_p; + auto bound_f = bound_p.get_future(); + auto unbound_f = unbound_p.get_future(); + long count = -1; + std::thread w([&] { + d->bind_current_thread(); + bound_p.set_value(); + unbound_f.wait(); // main thread has called unbind_thread() + cudaqx_affinity_syscall_reset(); + d->decode_batch(kChunk); + count = cudaqx_affinity_syscall_count(); + }); + bound_f.wait(); + d->unbind_thread(); // from the main thread, not the owner + unbound_p.set_value(); + w.join(); + EXPECT_GT(count, 0) + << "after unbind_thread() from another thread, the previously bound " + "thread must be guarded again"; +} + +// T3.16: two never-bound threads decode concurrently; each thread's placement +// must be fully restored and each decode correct. No shim-counter asserts: +// the counters are global and would race here. +TEST(PinningInvariants, ConcurrentUnboundDecodesRestoreBothThreads) { + auto d = makePinnedLut(); + std::atomic go{false}; + auto body = [&] { + while (!go.load(std::memory_order_acquire)) + std::this_thread::yield(); + auto before = thread_placement::capture(); + auto results = d->decode_batch(kChunk); + auto after = thread_placement::capture(); + EXPECT_EQ(before, after) << "concurrent unbound decode leaked placement: " + << before.describe_difference(after); + ASSERT_EQ(results.size(), kChunk.size()); + EXPECT_TRUE(results[0].converged); + }; + std::thread t1(body), t2(body); + go.store(true, std::memory_order_release); + t1.join(); + t2.join(); +} + +// T3.17: N threads race bind_current_thread() on one decoder. The race itself +// must not throw, and afterwards a FRESH never-bound thread must still be +// guarded and decode correctly. The racers are held alive (promise/shared +// handshake) until the fresh thread finishes so its thread id cannot be a +// recycled racer id that would spuriously skip the guard. No attempt to +// identify the winner. +TEST(PinningBenchmark, BindRaceLeavesFreshThreadsGuarded) { + if (!cudaqx_affinity_syscall_count) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + auto d = makePinnedLut(); + constexpr int kRacers = 4; + std::atomic go{false}; + std::atomic raced{0}; + std::promise done_p; + std::shared_future done_f(done_p.get_future()); + std::vector racers; + for (int i = 0; i < kRacers; ++i) + racers.emplace_back([&] { + while (!go.load(std::memory_order_acquire)) + std::this_thread::yield(); + EXPECT_NO_THROW(d->bind_current_thread()); + raced.fetch_add(1, std::memory_order_acq_rel); + done_f.wait(); // stay alive: our thread id must not be recycled yet + }); + go.store(true, std::memory_order_release); + while (raced.load(std::memory_order_acquire) < kRacers) + std::this_thread::yield(); + std::thread fresh([&] { + cudaqx_affinity_syscall_reset(); + auto results = d->decode_batch(kChunk); + EXPECT_GT(cudaqx_affinity_syscall_count(), 0) + << "a fresh thread that never bound must be guarded after the race"; + ASSERT_EQ(results.size(), kChunk.size()); + EXPECT_TRUE(results[0].converged); + }); + fresh.join(); + done_p.set_value(); + for (auto &t : racers) + t.join(); +} + +// ---- syscall / sysfs budget gates ------------------------------------------- + +// Perf-regression gate: the per-call guard must stay within a fixed syscall +// budget. If you legitimately add a syscall, change the constant in the same +// PR and say why in the commit message. +TEST(PinningBudget, UnboundDecodeStaysWithinSyscallBudget) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + run_on_worker([] { + auto d = makePinnedLut(); + (void)d->decode_batch(kChunk); // warm one-time paths + constexpr int kCalls = 10; + cudaqx_affinity_syscall_reset(); + for (int i = 0; i < kCalls; ++i) + (void)d->decode_batch(kChunk); + auto c = readShimCounts(); + long total = c.setaff + c.getaff + c.setmem + c.getmem; + std::printf( + "[pinning budget] %ld placement syscalls / %d unbound decodes " + "= %.1f per call (setaff=%ld getaff=%ld setmem=%ld getmem=%ld)\n", + total, kCalls, static_cast(total) / kCalls, c.setaff, c.getaff, + c.setmem, c.getmem); + // Measured on a mempolicy-capable host: 7 syscalls per unbound decode + // (2 sched_getaffinity + 2 sched_setaffinity + 1 get_mempolicy + + // 2 set_mempolicy). Budget = measured + one call of headroom. + constexpr long kMeasuredPerCall = 7; + EXPECT_LE(total, kMeasuredPerCall * kCalls + kMeasuredPerCall) + << "guard syscall budget exceeded: " << total << " for " << kCalls + << " calls"; + EXPECT_GT(total, 0); + }); +} + +// Sysfs budget gate: each guarded (unbound) decode currently re-reads the +// node's cpulist -- 1 open of /sys/devices/system/node/... per call. Cpuset +// caching should reduce this to amortized 0 -- tighten when it lands. +TEST(PinningBudget, UnboundDecodeStaysWithinSysfsOpenBudget) { + if (!shimCountersPresent() || !cudaqx_node_sysfs_open_count) + GTEST_SKIP() << "affinity-counter shim (with sysfs open counter) not " + "preloaded"; + run_on_worker([] { + auto d = makePinnedLut(); + (void)d->decode_batch(kChunk); // warm one-time paths + constexpr int kCalls = 10; + cudaqx_affinity_syscall_reset(); + for (int i = 0; i < kCalls; ++i) + (void)d->decode_batch(kChunk); + long opens = cudaqx_node_sysfs_open_count(); + std::printf("[pinning budget] %ld node-sysfs opens / %d unbound decodes " + "= %.1f per call\n", + opens, kCalls, static_cast(opens) / kCalls); + EXPECT_LE(opens, kCalls) << "node-sysfs open budget exceeded: " << opens + << " for " << kCalls << " calls"; + EXPECT_GT(opens, 0) << "expected the unbound guard to read the node " + "cpulist (counter wired?)"; + }); +} From f3ab07f21e36378022ad3f48014981062540d090 Mon Sep 17 00:00:00 2001 From: kvmto Date: Mon, 6 Jul 2026 22:07:10 +0000 Subject: [PATCH 12/16] fix(qec): close hardware-pinning coverage gaps across all surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YAML/config: mempolicy and cpu_affinity are now expressible in the realtime config (struct fields, YAML mapping, prepare_decoder_params, Python config bindings) — all four placement knobs exist on every surface. Core: RAII guards move to a shared lib-private header; public mempolicy() reader (trt stops reading the raw member); guards hoisted above input processing in decode(tensor) and trt's padding path; the sparse-matrix setters guard their session-lifetime allocations; new non-virtual decode_guarded() for single-syndrome callers that have not bound a thread. Realtime session: placement consensus covers all four knobs and engages on any of device/node/cpu list; conflicting mempolicy warns; DEVICE mode runs graph capture, the dispatch kernel, stats allocation, and worker streams on the decoders' agreed device and rejects conflicting devices loudly at initialize; worker streams drain before teardown frees graph buffers; ring buffers, function tables, and control words first-touch on the session node. Python: decode routes through the guarded entry; bind_current_thread/ unbind_thread and mempolicy/cpu_affinity readbacks are exposed; classes overriding decode_batch/decode_async in Python are rejected loudly when placement kwargs are passed, since attribute lookup would bypass the guarded C++ entry points. Signed-off-by: kvmto --- libs/qec/include/cudaq/qec/decoder.h | 10 + .../cudaq/qec/realtime/decoding_config.h | 2 + libs/qec/lib/decoder.cpp | 145 +++--------- .../plugins/trt_decoder/trt_decoder.cpp | 32 ++- libs/qec/lib/hardware_guards.h | 127 +++++++++++ libs/qec/lib/realtime/CMakeLists.txt | 2 +- libs/qec/lib/realtime/config.cpp | 2 + .../qec/lib/realtime/qec_realtime_session.cpp | 215 ++++++++++++------ libs/qec/lib/realtime/qec_realtime_session.h | 17 +- libs/qec/lib/realtime/realtime_decoding.cpp | 4 + libs/qec/python/bindings/py_decoder.cpp | 65 +++++- .../python/bindings/py_decoding_config.cpp | 2 + libs/qec/python/tests/test_decoder.py | 42 ++++ .../pymatching/test_pymatching_realtime.cpp | 74 ++++++ libs/qec/unittests/test_decoders_yaml.cpp | 32 +++ libs/qec/unittests/test_pinning_benchmark.cpp | 86 +++++++ 16 files changed, 653 insertions(+), 204 deletions(-) create mode 100644 libs/qec/lib/hardware_guards.h diff --git a/libs/qec/include/cudaq/qec/decoder.h b/libs/qec/include/cudaq/qec/decoder.h index 0958e6bda..0662d3a24 100644 --- a/libs/qec/include/cudaq/qec/decoder.h +++ b/libs/qec/include/cudaq/qec/decoder.h @@ -268,6 +268,10 @@ class decoder /// @brief Target CUDA device for this decoder (-1 = inherit caller's device). int cuda_device_id() const { return cuda_device_id_; } + /// @brief NUMA memory-policy mode for this decoder (preferred = soft, + /// bind = strict). + cudaq::qec::mempolicy_mode mempolicy() const { return mempolicy_; } + /// @brief Explicit CPU list for this decoder (empty = derive from NUMA node). const std::vector &cpu_affinity() const { return cpu_affinity_; } @@ -284,6 +288,12 @@ class decoder /// throughput drive decode on a long-lived bound thread instead. decoder_result decode_on_pinned_thread(const std::vector &syndrome); + /// @brief Single-syndrome decode with the same automatic CUDA/NUMA guard as + /// decode_batch(). Non-virtual; dispatches to the virtual decode() inside + /// the guard. Intended for host-language bindings and callers that cannot + /// use bind_current_thread(); bound threads skip the guard as usual. + decoder_result decode_guarded(const std::vector &syndrome); + /// @brief Forget any bind_current_thread() registration on this decoder so /// guarded entry points stop skipping their per-call guard. Call before the /// bound thread exits (e.g. session teardown): thread ids are recycled by diff --git a/libs/qec/include/cudaq/qec/realtime/decoding_config.h b/libs/qec/include/cudaq/qec/realtime/decoding_config.h index e078d690f..d7f02a45d 100644 --- a/libs/qec/include/cudaq/qec/realtime/decoding_config.h +++ b/libs/qec/include/cudaq/qec/realtime/decoding_config.h @@ -181,6 +181,8 @@ struct decoder_config { std::vector D_sparse; std::optional cuda_device_id; std::optional numa_node_id; + std::optional mempolicy; // "preferred" | "bind" + std::optional> cpu_affinity; // explicit core ids std::variant diff --git a/libs/qec/lib/decoder.cpp b/libs/qec/lib/decoder.cpp index 57f744ea4..85d5bdb62 100644 --- a/libs/qec/lib/decoder.cpp +++ b/libs/qec/lib/decoder.cpp @@ -31,117 +31,10 @@ #include #endif #include "hardware_affinity.h" +#include "hardware_guards.h" -namespace { - -// RAII: sets the calling thread's CUDA current device, restores on destruction. -// target < 0 = no-op. Throws std::runtime_error if: target is out of range -// of the visible device count, cudaGetDevice() fails while checking the -// current device, or cudaSetDevice() fails while switching to target. -struct CudaDeviceGuard { - int prev_ = -1; - bool active_ = false; - - explicit CudaDeviceGuard(int target) { - if (target < 0) - return; - int count = 0; - if (cudaGetDeviceCount(&count) != cudaSuccess || target >= count) - throw std::runtime_error( - "cuda_device_id " + std::to_string(target) + - " out of range (device_count=" + std::to_string(count) + ")"); - if (cudaGetDevice(&prev_) != cudaSuccess) { - // Can't determine the current device; warn and skip the switch so the - // decode proceeds on whatever device the caller is already on. - cudaq::qec::detail_affinity::affinity_warn( - "CudaDeviceGuard: cudaGetDevice failed for cuda_device_id " + - std::to_string(target) + "; skipping device switch"); - return; - } - if (prev_ != target) { - cudaError_t e = cudaSetDevice(target); - if (e != cudaSuccess) - throw std::runtime_error("CudaDeviceGuard: cudaSetDevice(" + - std::to_string(target) + - ") failed: " + cudaGetErrorString(e)); - active_ = true; - } - } - ~CudaDeviceGuard() { - if (!active_) - return; - if (cudaSetDevice(prev_) != cudaSuccess) - cudaq::qec::detail_affinity::affinity_warn( - "CudaDeviceGuard: failed to restore prior CUDA device " + - std::to_string(prev_) + "; thread may remain on wrong device"); - } - CudaDeviceGuard(const CudaDeviceGuard &) = delete; - CudaDeviceGuard &operator=(const CudaDeviceGuard &) = delete; -}; - -#if defined(__linux__) -// RAII: binds the calling thread to a NUMA node and restores on destruction. -// node < 0 = no-op. Uses the shared persistent-bind primitive for the set half -// and remembers prior affinity for the restore. -struct NumaGuard { - bool affinity_set_ = false; - bool has_prev_affinity_ = false; - bool mempol_set_ = false; - cpu_set_t prev_set_{}; - cudaq::qec::detail_affinity::mempolicy_state prev_mempolicy_; - - explicit NumaGuard(int node, cudaq::qec::mempolicy_mode mode = - cudaq::qec::mempolicy_mode::preferred) { - if (node < 0) - return; - CPU_ZERO(&prev_set_); - has_prev_affinity_ = - (sched_getaffinity(0, sizeof(prev_set_), &prev_set_) == 0); - if (node < static_cast(sizeof(unsigned long) * 8)) { - prev_mempolicy_ = cudaq::qec::detail_affinity::capture_thread_mempolicy(); - mempol_set_ = (prev_mempolicy_.mode >= 0); - if (!mempol_set_) - cudaq::qec::detail_affinity::affinity_warn( - "NumaGuard: prior thread mempolicy unreadable (get_mempolicy " - "failed); temporary NUMA memory policy skipped for this decode"); - } - // A temporary guard must never apply a policy it cannot restore: skip the - // mempolicy half when the capture failed (affinity is still applied, its - // restore path is independent). - cudaq::qec::detail_affinity::bind_this_thread_to_numa_node( - node, mode, /*apply_mempolicy=*/mempol_set_); - // Arm the affinity restore only after a successful bind; if bind throws the - // affinity was not changed and there is nothing to undo. - affinity_set_ = has_prev_affinity_; - } - - ~NumaGuard() { - if (mempol_set_) - cudaq::qec::detail_affinity::restore_thread_mempolicy(prev_mempolicy_); - // has_prev_affinity_ is implied: it is set iff affinity_set_ is set. - if (affinity_set_) - if (sched_setaffinity(0, sizeof(prev_set_), &prev_set_) != 0) - cudaq::qec::detail_affinity::affinity_warn( - "NumaGuard restore: failed to reset thread affinity: " + - std::string(std::strerror(errno)) + "; thread may remain pinned"); - } - - NumaGuard(const NumaGuard &) = delete; - NumaGuard &operator=(const NumaGuard &) = delete; -}; -#else -struct NumaGuard { - explicit NumaGuard(int node, cudaq::qec::mempolicy_mode mode = - cudaq::qec::mempolicy_mode::preferred) { - (void)mode; - if (node < 0) - return; - cudaq::qec::detail_affinity::warn_numa_unsupported_once(); - } -}; -#endif - -} // anonymous namespace +using cudaq::qec::detail_affinity::CudaDeviceGuard; +using cudaq::qec::detail_affinity::NumaGuard; INSTANTIATE_REGISTRY(cudaq::qec::decoder, const cudaq::qec::decoder_init &, const cudaqx::heterogeneous_map &) @@ -379,6 +272,12 @@ decoder::decode_on_pinned_thread(const std::vector &syndrome) { // Provide a trivial implementation of for tensor decode call. Child // classes should override this if they never want to pass through floats. decoder_result decoder::decode(const cudaqx::tensor &syndrome) { + // Guards are constructed before ANY input processing so the soft-syndrome + // temporaries below are allocated on the decoder's device/NUMA node. They + // are exception-safe RAII, so placement is restored even on the rank throw. + const bool skip_guard = is_bound_here(); + CudaDeviceGuard dev(skip_guard ? -1 : cuda_device_id_); + NumaGuard numa(skip_guard ? -1 : numa_node_id_, mempolicy_); // Check tensor is of order-1 // If order >1, we could check that other modes are of dim = 1 such that // n x 1, or 1 x n tensors are still valid. @@ -389,10 +288,14 @@ decoder_result decoder::decode(const cudaqx::tensor &syndrome) { std::vector vec_cast(syndrome.data(), syndrome.data() + syndrome.shape()[0]); convert_vec_hard_to_soft(vec_cast, soft_syndrome); + return decode(soft_syndrome); +} + +decoder_result decoder::decode_guarded(const std::vector &syndrome) { const bool skip_guard = is_bound_here(); CudaDeviceGuard dev(skip_guard ? -1 : cuda_device_id_); NumaGuard numa(skip_guard ? -1 : numa_node_id_, mempolicy_); - return decode(soft_syndrome); + return decode(syndrome); } // Provide a trivial implementation of the multi-syndrome decoder. Child classes @@ -425,6 +328,8 @@ std::future decoder::decode_async(const std::vector &syndrome) { // The three affinity scalars are captured by value so the lambda doesn't // dereference decoder members on the hot path. + // The syndrome copy is captured on the CALLING thread; its pages follow the + // caller's placement. The worker's guards cover everything decode-side. // LIFETIME PRECONDITION: the decoder must outlive the returned future. // Destroying the decoder before calling .get() is undefined behaviour. const int cuda_id = cuda_device_id_; @@ -539,6 +444,10 @@ set_sparse_from_vec(const std::vector &vec_in, } void decoder::set_O_sparse(const std::vector> &O_sparse) { + // The corrections buffer allocated here lives for the whole session, so it + // must follow the decoder's placement (host allocation: no device guard). + const bool skip_guard = is_bound_here(); + NumaGuard numa(skip_guard ? -1 : numa_node_id_, mempolicy_); this->O_sparse = O_sparse; validate_sparse_column_indices(this->O_sparse, block_size, "O_sparse"); this->pimpl->corrections.clear(); @@ -546,6 +455,10 @@ void decoder::set_O_sparse(const std::vector> &O_sparse) { } void decoder::set_O_sparse(const std::vector &O_sparse_vec_in) { + // The corrections buffer allocated here lives for the whole session, so it + // must follow the decoder's placement (host allocation: no device guard). + const bool skip_guard = is_bound_here(); + NumaGuard numa(skip_guard ? -1 : numa_node_id_, mempolicy_); set_sparse_from_vec(O_sparse_vec_in, this->O_sparse); validate_sparse_column_indices(this->O_sparse, block_size, "O_sparse"); this->pimpl->corrections.clear(); @@ -595,11 +508,21 @@ void set_D_sparse_common(decoder *decoder, } void decoder::set_D_sparse(const std::vector> &D_sparse) { + // The msyn/detector buffers allocated here live for the whole session, so + // they must follow the decoder's placement (host allocation: no device + // guard). + const bool skip_guard = is_bound_here(); + NumaGuard numa(skip_guard ? -1 : numa_node_id_, mempolicy_); this->D_sparse = D_sparse; set_D_sparse_common(this, D_sparse, pimpl.get()); } void decoder::set_D_sparse(const std::vector &D_sparse_vec_in) { + // The msyn/detector buffers allocated here live for the whole session, so + // they must follow the decoder's placement (host allocation: no device + // guard). + const bool skip_guard = is_bound_here(); + NumaGuard numa(skip_guard ? -1 : numa_node_id_, mempolicy_); set_sparse_from_vec(D_sparse_vec_in, this->D_sparse); set_D_sparse_common(this, this->D_sparse, pimpl.get()); } diff --git a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp index 6a1bb3d41..ef2bc11f2 100644 --- a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp +++ b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp @@ -23,6 +23,7 @@ #include #endif #include "hardware_affinity.h" +#include "hardware_guards.h" // TensorRT headers #include "NvInfer.h" @@ -854,17 +855,26 @@ decoder_result trt_decoder::decode(const std::vector &syndrome) { "Model has batch_size={}, zero-padding single syndrome to fill batch", model_batch_size_); - // Create a batch with the real syndrome plus zero-padded syndromes + // Create a batch with the real syndrome plus zero-padded syndromes. + // Declared before the guard scope (the empty vector allocates nothing); + // every allocation happens inside the scope below. std::vector> padded_batch; - padded_batch.reserve(model_batch_size_); - - // First syndrome is the real one - padded_batch.push_back(syndrome); - - // Fill remaining batch slots with zero syndromes - std::vector zero_syndrome(syndrome_size_per_sample_, 0.0f); - for (size_t i = 1; i < model_batch_size_; ++i) { - padded_batch.push_back(zero_syndrome); + { + // The padding buffers must land on the decoder's NUMA node like every + // other decode-side allocation. The guard scope closes before the + // delegation because decode_batch() re-applies its own guards. + cudaq::qec::detail_affinity::NumaGuard place( + is_bound_here() ? -1 : numa_node_id_, mempolicy()); + padded_batch.reserve(model_batch_size_); + + // First syndrome is the real one + padded_batch.push_back(syndrome); + + // Fill remaining batch slots with zero syndromes + std::vector zero_syndrome(syndrome_size_per_sample_, 0.0f); + for (size_t i = 1; i < model_batch_size_; ++i) { + padded_batch.push_back(zero_syndrome); + } } auto results = decode_batch(padded_batch); @@ -970,7 +980,7 @@ trt_decoder::decode_batch(const std::vector> &syndromes) { "thread may remain pinned"); #endif } - } _numa_guard(skip_guard ? -1 : numa_node_id_, mempolicy_); + } _numa_guard(skip_guard ? -1 : numa_node_id_, mempolicy()); // Validate that we have syndromes to decode if (syndromes.empty()) { diff --git a/libs/qec/lib/hardware_guards.h b/libs/qec/lib/hardware_guards.h new file mode 100644 index 000000000..93fc8dea0 --- /dev/null +++ b/libs/qec/lib/hardware_guards.h @@ -0,0 +1,127 @@ +/******************************************************************************* + * Copyright (c) 2025 - 2026 NVIDIA Corporation & Affiliates. * + * All rights reserved. * + * * + * This source code and the accompanying materials are made available under * + * the terms of the Apache License 2.0 which accompanies this distribution. * + ******************************************************************************/ + +// Lib-private (NOT installed): RAII guards that place the calling thread on a +// decoder's CUDA device / NUMA node and restore on scope exit. Shared by the +// base decoder entry points and the realtime session. Never include from a +// public header. +#pragma once + +#include "hardware_affinity.h" +#include + +namespace cudaq::qec::detail_affinity { + +// RAII: sets the calling thread's CUDA current device, restores on destruction. +// target < 0 = no-op. Throws std::runtime_error if: target is out of range +// of the visible device count, cudaGetDevice() fails while checking the +// current device, or cudaSetDevice() fails while switching to target. +struct CudaDeviceGuard { + int prev_ = -1; + bool active_ = false; + + explicit CudaDeviceGuard(int target) { + if (target < 0) + return; + int count = 0; + if (cudaGetDeviceCount(&count) != cudaSuccess || target >= count) + throw std::runtime_error( + "cuda_device_id " + std::to_string(target) + + " out of range (device_count=" + std::to_string(count) + ")"); + if (cudaGetDevice(&prev_) != cudaSuccess) { + // Can't determine the current device; warn and skip the switch so the + // decode proceeds on whatever device the caller is already on. + cudaq::qec::detail_affinity::affinity_warn( + "CudaDeviceGuard: cudaGetDevice failed for cuda_device_id " + + std::to_string(target) + "; skipping device switch"); + return; + } + if (prev_ != target) { + cudaError_t e = cudaSetDevice(target); + if (e != cudaSuccess) + throw std::runtime_error("CudaDeviceGuard: cudaSetDevice(" + + std::to_string(target) + + ") failed: " + cudaGetErrorString(e)); + active_ = true; + } + } + ~CudaDeviceGuard() { + if (!active_) + return; + if (cudaSetDevice(prev_) != cudaSuccess) + cudaq::qec::detail_affinity::affinity_warn( + "CudaDeviceGuard: failed to restore prior CUDA device " + + std::to_string(prev_) + "; thread may remain on wrong device"); + } + CudaDeviceGuard(const CudaDeviceGuard &) = delete; + CudaDeviceGuard &operator=(const CudaDeviceGuard &) = delete; +}; + +#if defined(__linux__) +// RAII: binds the calling thread to a NUMA node and restores on destruction. +// node < 0 = no-op. Uses the shared persistent-bind primitive for the set half +// and remembers prior affinity for the restore. +struct NumaGuard { + bool affinity_set_ = false; + bool has_prev_affinity_ = false; + bool mempol_set_ = false; + cpu_set_t prev_set_{}; + cudaq::qec::detail_affinity::mempolicy_state prev_mempolicy_; + + explicit NumaGuard(int node, cudaq::qec::mempolicy_mode mode = + cudaq::qec::mempolicy_mode::preferred) { + if (node < 0) + return; + CPU_ZERO(&prev_set_); + has_prev_affinity_ = + (sched_getaffinity(0, sizeof(prev_set_), &prev_set_) == 0); + if (node < static_cast(sizeof(unsigned long) * 8)) { + prev_mempolicy_ = cudaq::qec::detail_affinity::capture_thread_mempolicy(); + mempol_set_ = (prev_mempolicy_.mode >= 0); + if (!mempol_set_) + cudaq::qec::detail_affinity::affinity_warn( + "NumaGuard: prior thread mempolicy unreadable (get_mempolicy " + "failed); temporary NUMA memory policy skipped for this decode"); + } + // A temporary guard must never apply a policy it cannot restore: skip the + // mempolicy half when the capture failed (affinity is still applied, its + // restore path is independent). + cudaq::qec::detail_affinity::bind_this_thread_to_numa_node( + node, mode, /*apply_mempolicy=*/mempol_set_); + // Arm the affinity restore only after a successful bind; if bind throws the + // affinity was not changed and there is nothing to undo. + affinity_set_ = has_prev_affinity_; + } + + ~NumaGuard() { + if (mempol_set_) + cudaq::qec::detail_affinity::restore_thread_mempolicy(prev_mempolicy_); + // has_prev_affinity_ is implied: it is set iff affinity_set_ is set. + if (affinity_set_) + if (sched_setaffinity(0, sizeof(prev_set_), &prev_set_) != 0) + cudaq::qec::detail_affinity::affinity_warn( + "NumaGuard restore: failed to reset thread affinity: " + + std::string(std::strerror(errno)) + "; thread may remain pinned"); + } + + NumaGuard(const NumaGuard &) = delete; + NumaGuard &operator=(const NumaGuard &) = delete; +}; +#else +struct NumaGuard { + explicit NumaGuard(int node, cudaq::qec::mempolicy_mode mode = + cudaq::qec::mempolicy_mode::preferred) { + (void)mode; + if (node < 0) + return; + cudaq::qec::detail_affinity::warn_numa_unsupported_once(); + } +}; +#endif + +} // namespace cudaq::qec::detail_affinity diff --git a/libs/qec/lib/realtime/CMakeLists.txt b/libs/qec/lib/realtime/CMakeLists.txt index 6bc8c4b16..ab387a1e1 100644 --- a/libs/qec/lib/realtime/CMakeLists.txt +++ b/libs/qec/lib/realtime/CMakeLists.txt @@ -6,7 +6,7 @@ # the terms of the Apache License 2.0 which accompanies this distribution. # # ============================================================================ # -find_package(LLVM 22.1.4 EXACT REQUIRED CONFIG) +find_package(LLVM 22.1 REQUIRED CONFIG) # Enable CUDA for device code if(CMAKE_CUDA_COMPILER) diff --git a/libs/qec/lib/realtime/config.cpp b/libs/qec/lib/realtime/config.cpp index 46f78357f..bd5018204 100644 --- a/libs/qec/lib/realtime/config.cpp +++ b/libs/qec/lib/realtime/config.cpp @@ -711,6 +711,8 @@ struct MappingTraits { io.mapRequired("D_sparse", config.D_sparse); io.mapOptional("cuda_device_id", config.cuda_device_id); io.mapOptional("numa_node_id", config.numa_node_id); + io.mapOptional("mempolicy", config.mempolicy); + io.mapOptional("cpu_affinity", config.cpu_affinity); // Validate that the number of rows in the H_sparse vector is equal to // syndrome_size. diff --git a/libs/qec/lib/realtime/qec_realtime_session.cpp b/libs/qec/lib/realtime/qec_realtime_session.cpp index 5004ee47b..f9ca55a65 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.cpp +++ b/libs/qec/lib/realtime/qec_realtime_session.cpp @@ -10,8 +10,10 @@ #include "qec_realtime_session.h" -// Lib-private header one level up (libs/qec/lib); this file lives in realtime/. +// Lib-private headers one level up (libs/qec/lib); this file lives in +// realtime/. #include "../hardware_affinity.h" +#include "../hardware_guards.h" #include "cudaq/qec/logger.h" #include "cudaq/qec/realtime/decoder_rpc_ids.h" #include "cudaq/qec/realtime/graph_resources.h" @@ -330,11 +332,11 @@ void qec_realtime_session::initialize() { // finalize() is null-safe at every step, so we can roll a half-built session // back from any throw. try { - if (device_mode_) - capture_decoder_graphs(); - // Resolve the session NUMA node from the decoders. A single shared dispatch - // thread can honor only one node; warn and disable pinning if they - // disagree. + // Resolve the session's placement consensus (NUMA node, CUDA device, CPU + // list, mempolicy) from the decoders before the CUDA resource setup + // below: graph capture must run on the agreed device. A single shared + // dispatch thread can honor only one of each; warn and disable pinning + // on disagreement. { int chosen = -1; bool conflict = false; @@ -342,6 +344,10 @@ void qec_realtime_session::initialize() { bool dev_conflict = false; std::vector chosen_cpus; bool affinity_conflict = false; + bool have_mempolicy = false; + bool mempolicy_conflict = false; + cudaq::qec::mempolicy_mode chosen_mempolicy = + cudaq::qec::mempolicy_mode::preferred; for (const auto &d : decoders_) { if (!d) continue; @@ -369,6 +375,16 @@ void qec_realtime_session::initialize() { else if (chosen_cpus != cpus) affinity_conflict = true; } + // mempolicy participates only when the decoder set a placement knob; + // the default (preferred) never conflicts with itself. + if (n >= 0 || dev >= 0 || !cpus.empty()) { + if (!have_mempolicy) { + chosen_mempolicy = d->mempolicy(); + have_mempolicy = true; + } else if (chosen_mempolicy != d->mempolicy()) { + mempolicy_conflict = true; + } + } } if (conflict) { CUDA_QEC_WARN( @@ -379,7 +395,9 @@ void qec_realtime_session::initialize() { } else { session_numa_node_ = chosen; } - if (dev_conflict) + // HOST mode only: DEVICE mode throws on dev_conflict below, and that + // message supersedes this one (per-call guards never get to run there). + if (dev_conflict && !device_mode_) CUDA_QEC_WARN( "realtime decoders request different cuda_device_ids; the shared " "dispatch thread cannot hold one device for all of them. " @@ -389,12 +407,32 @@ void qec_realtime_session::initialize() { "realtime decoders request different cpu_affinity lists; the " "shared dispatch thread can honor only one. Guard-skip binding " "disabled; per-call guards stay active."); - bind_decoders_to_host_loop_ = !dev_conflict && !conflict && - !affinity_conflict && - (chosen >= 0 || chosen_dev >= 0); + if (mempolicy_conflict) + CUDA_QEC_WARN( + "realtime decoders request different mempolicy modes; the shared " + "dispatch thread can honor only one. Guard-skip binding disabled; " + "per-call guards stay active."); + bind_decoders_to_host_loop_ = + !dev_conflict && !conflict && !affinity_conflict && + !mempolicy_conflict && + (chosen >= 0 || chosen_dev >= 0 || !chosen_cpus.empty()); + session_cuda_device_ = dev_conflict ? -1 : chosen_dev; + if (device_mode_ && dev_conflict) + throw std::runtime_error( + "qec_realtime_session::initialize: decoders request different " + "cuda_device_ids; DEVICE mode runs one dispatch kernel on one " + "device and cannot honor them. Use one device per session."); + } + if (device_mode_) + capture_decoder_graphs(); + { + // Ring buffers, function tables, and control words are polled every + // dispatch iteration; first-touch them on the session node. Scoped: + // restores the config thread's placement before the loops start. + cudaq::qec::detail_affinity::NumaGuard place(session_numa_node_); + allocate_ring_buffer(); + populate_function_table(); } - allocate_ring_buffer(); - populate_function_table(); if (device_mode_) start_device_loop(); start_host_loop(); @@ -516,6 +554,11 @@ void qec_realtime_session::finalize() { //============================================================================== void qec_realtime_session::capture_decoder_graphs() { + // Graph capture allocates streams/exec graphs on the CURRENT device; place + // it on the session's agreed device so a pinned decoder's graph does not + // land on the ambient device of whichever thread called initialize(). + cudaq::qec::detail_affinity::CudaDeviceGuard place(session_cuda_device_); + captured_graphs_.assign(decoders_.size(), nullptr); num_decoders_with_graph_ = 0; @@ -846,13 +889,18 @@ void qec_realtime_session::populate_function_table() { //============================================================================== void qec_realtime_session::start_device_loop() { + // Dispatcher setup below (including the device_stats_dev_ cudaMalloc) runs + // on the CURRENT device; place it on the session's agreed device so the + // persistent dispatch kernel's resources live with the captured graphs. + cudaq::qec::detail_affinity::CudaDeviceGuard place(session_cuda_device_); + if (cudaq_dispatch_manager_create(&device_manager_) != CUDAQ_OK) throw std::runtime_error( "qec_realtime_session::initialize: cudaq_dispatch_manager_create " "failed"); cudaq_dispatcher_config_t dev_config{}; - dev_config.device_id = 0; + dev_config.device_id = session_cuda_device_ >= 0 ? session_cuda_device_ : 0; dev_config.num_blocks = 1; dev_config.threads_per_block = 64; dev_config.num_slots = static_cast(num_slots_); @@ -943,8 +991,10 @@ void qec_realtime_session::start_host_loop() { // Persistent bind via decoder::bind_current_thread() so that // is_bound_here() returns true and per-call NUMA/CUDA guards are // suppressed in the decode hot path. Only when initialize() proved all - // decoders agree on both placement knobs; otherwise keep per-call - // guards active and give the thread plain NUMA locality at most. + // decoders agree on all placement knobs (device, node, cpu list, + // mempolicy); agreement on any knob engages the bind. Otherwise keep + // per-call guards active and give the thread plain NUMA locality at + // most. if (bind_decoders) { for (auto &d : decoders_) { if (!d) @@ -993,60 +1043,72 @@ void qec_realtime_session::start_host_loop() { void **mailbox_bank = nullptr; - std::size_t slot = 0; - for (std::size_t i = 0; i < decoders_.size(); ++i) { - if (!captured_graphs_[i]) - continue; - auto *gres = static_cast( - captured_graphs_[i]); - - cudaStream_t stream = nullptr; - if (cudaStreamCreate(&stream) != cudaSuccess) - throw std::runtime_error( - "qec_realtime_session::initialize: cudaStreamCreate for HOST_LOOP " - "worker " + - std::to_string(slot) + " (decoder_id=" + std::to_string(i) + - ") failed"); - host_worker_streams_[i] = stream; - - auto &w = host_workers_[slot]; - w.graph_exec = gres->graph_exec; - w.stream = stream; - w.function_id = cudaq::qec::decoding::rpc::kEnqueueSyndromesFunctionId; - w.routing_key = static_cast(i); - w.pre_launch_fn = nullptr; - w.pre_launch_data = nullptr; - w.post_launch_fn = nullptr; - w.post_launch_data = nullptr; - - if (slot == 0) - mailbox_bank = gres->h_mailbox; - ++slot; - } - - host_idle_mask_storage_ = new std::uint64_t( - num_decoders_with_graph_ < 64 - ? ((std::uint64_t{1} << num_decoders_with_graph_) - 1) - : ~std::uint64_t{0}); - host_live_dispatched_storage_ = new std::uint64_t(0); - host_inflight_slot_tags_ = new int[num_decoders_with_graph_]; - for (std::size_t i = 0; i < num_decoders_with_graph_; ++i) - host_inflight_slot_tags_[i] = -1; - - // Per-worker GraphIOContext array (pinned-mapped so both CPU monitor and GPU - // graph see the same backing). { - void *h = nullptr; - void *d = nullptr; - const std::size_t bytes = - num_decoders_with_graph_ * sizeof(cudaq::realtime::GraphIOContext); - if (!allocate_pinned_mapped(bytes, &h, &d)) - throw std::runtime_error("qec_realtime_session::start_host_loop: failed " - "to allocate per-worker " - "GraphIOContext array"); - std::memset(h, 0, bytes); - io_ctxs_host_ = static_cast(h); - io_ctxs_dev_ = static_cast(d); + // Worker streams and the pinned-mapped GraphIOContext array are created + // on the CURRENT device; place them on the session's agreed device so + // they live where the captured graphs run (no-op when no decoder pinned + // a device). The host-visible halves (GraphIOContext array, control + // words) are first-touched here, so also place them on the session node. + // The host_loop_thread_ spawned below stays outside this guard: it binds + // itself. + cudaq::qec::detail_affinity::CudaDeviceGuard place(session_cuda_device_); + cudaq::qec::detail_affinity::NumaGuard numa_place(session_numa_node_); + + std::size_t slot = 0; + for (std::size_t i = 0; i < decoders_.size(); ++i) { + if (!captured_graphs_[i]) + continue; + auto *gres = static_cast( + captured_graphs_[i]); + + cudaStream_t stream = nullptr; + if (cudaStreamCreate(&stream) != cudaSuccess) + throw std::runtime_error( + "qec_realtime_session::initialize: cudaStreamCreate for HOST_LOOP " + "worker " + + std::to_string(slot) + " (decoder_id=" + std::to_string(i) + + ") failed"); + host_worker_streams_[i] = stream; + + auto &w = host_workers_[slot]; + w.graph_exec = gres->graph_exec; + w.stream = stream; + w.function_id = cudaq::qec::decoding::rpc::kEnqueueSyndromesFunctionId; + w.routing_key = static_cast(i); + w.pre_launch_fn = nullptr; + w.pre_launch_data = nullptr; + w.post_launch_fn = nullptr; + w.post_launch_data = nullptr; + + if (slot == 0) + mailbox_bank = gres->h_mailbox; + ++slot; + } + + host_idle_mask_storage_ = new std::uint64_t( + num_decoders_with_graph_ < 64 + ? ((std::uint64_t{1} << num_decoders_with_graph_) - 1) + : ~std::uint64_t{0}); + host_live_dispatched_storage_ = new std::uint64_t(0); + host_inflight_slot_tags_ = new int[num_decoders_with_graph_]; + for (std::size_t i = 0; i < num_decoders_with_graph_; ++i) + host_inflight_slot_tags_[i] = -1; + + // Per-worker GraphIOContext array (pinned-mapped so both CPU monitor and + // GPU graph see the same backing). + { + void *h = nullptr; + void *d = nullptr; + const std::size_t bytes = + num_decoders_with_graph_ * sizeof(cudaq::realtime::GraphIOContext); + if (!allocate_pinned_mapped(bytes, &h, &d)) + throw std::runtime_error( + "qec_realtime_session::start_host_loop: failed to allocate " + "per-worker GraphIOContext array"); + std::memset(h, 0, bytes); + io_ctxs_host_ = static_cast(h); + io_ctxs_dev_ = static_cast(d); + } } std::memset(&host_ctx_, 0, sizeof(host_ctx_)); @@ -1081,9 +1143,10 @@ void qec_realtime_session::start_host_loop() { host_loop_thread_ = std::thread([this, node, bind_decoders]() { // Mirror HOST-mode: bind via decoder::bind_current_thread() so that // guard-skip is activated for the decode hot path. Only when - // initialize() proved all decoders agree on both placement knobs; - // otherwise keep per-call guards active and give the thread plain NUMA - // locality at most. + // initialize() proved all decoders agree on all placement knobs + // (device, node, cpu list, mempolicy); agreement on any knob engages + // the bind. Otherwise keep per-call guards active and give the thread + // plain NUMA locality at most. if (bind_decoders) { for (auto &d : decoders_) { if (!d) @@ -1148,6 +1211,16 @@ void qec_realtime_session::stop_loops() { device_manager_ = nullptr; } + // Drain every worker stream before anything frees graph IO buffers: + // stream handles carry their device, so this synchronizes correctly even + // when the tearing-down thread's current device differs. In-flight graph + // launches from the non-blocking submit path must complete before + // release_decode_graph()/finalize() free the memory they read. + for (auto s : host_worker_streams_) { + if (s) + cudaStreamSynchronize(s); + } + for (auto s : host_worker_streams_) { if (s) cudaStreamDestroy(s); diff --git a/libs/qec/lib/realtime/qec_realtime_session.h b/libs/qec/lib/realtime/qec_realtime_session.h index 1a1084ce6..9f1204feb 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.h +++ b/libs/qec/lib/realtime/qec_realtime_session.h @@ -211,11 +211,18 @@ class __attribute__((visibility("default"))) qec_realtime_session { /// Single NUMA node this session pins to (dispatch thread + ring buffers). /// -1 = no pinning (unset, or decoders disagree — see initialize()). int session_numa_node_ = -1; - /// True when every decoder that sets a placement knob agrees on BOTH - /// cuda_device_id and numa_node_id — only then may the shared dispatch - /// thread register itself via bind_current_thread() (a bound decoder skips - /// its per-call guard, so binding under a device conflict would decode on - /// the wrong GPU). + /// Single CUDA device this session's decoders agreed on (-1 = none set, + /// or decoders disagree in HOST mode). DEVICE mode captures graphs, creates + /// worker streams, and launches the dispatch kernel on this device; + /// conflicting devices are rejected there. + int session_cuda_device_ = -1; + /// True when every decoder that sets a placement knob agrees on ALL of + /// cuda_device_id, numa_node_id, cpu_affinity, and mempolicy, and at least + /// one of numa_node_id / cuda_device_id / cpu_affinity is actually set — + /// only then may the shared dispatch thread register itself via + /// bind_current_thread() (a bound decoder skips its per-call guard, so + /// binding under any placement conflict would run some decoder on the + /// wrong GPU, node, CPU set, or memory policy). bool bind_decoders_to_host_loop_ = false; std::uint64_t host_stats_counter_ = 0; // Plain (non-pinned) shutdown flag for HOST mode (no device kernel shares diff --git a/libs/qec/lib/realtime/realtime_decoding.cpp b/libs/qec/lib/realtime/realtime_decoding.cpp index c487046e8..5471dc164 100644 --- a/libs/qec/lib/realtime/realtime_decoding.cpp +++ b/libs/qec/lib/realtime/realtime_decoding.cpp @@ -187,6 +187,10 @@ cudaqx::heterogeneous_map prepare_decoder_params( params.insert("cuda_device_id", decoder_config.cuda_device_id.value()); if (decoder_config.numa_node_id.has_value()) params.insert("numa_node_id", decoder_config.numa_node_id.value()); + if (decoder_config.mempolicy.has_value()) + params.insert("mempolicy", decoder_config.mempolicy.value()); + if (decoder_config.cpu_affinity.has_value()) + params.insert("cpu_affinity", decoder_config.cpu_affinity.value()); if (decoder_config.type != "trt_decoder") return params; diff --git a/libs/qec/python/bindings/py_decoder.cpp b/libs/qec/python/bindings/py_decoder.cpp index ce091bc51..85d7f501b 100644 --- a/libs/qec/python/bindings/py_decoder.cpp +++ b/libs/qec/python/bindings/py_decoder.cpp @@ -222,13 +222,20 @@ class PyDecoderRegistry { static std::unordered_map> registry; + // Whether the registered class defines decode_batch/decode_async in Python + // (recorded at registration time from the class namespace). Such overrides + // are dispatched by Python attribute lookup, bypassing the guarded C++ + // entry points, so hardware placement kwargs cannot take effect on them. + static std::unordered_map shadow_flags; public: static void register_decoder(const std::string &name, - std::function factory) { + std::function factory, + bool shadows_guarded_entries) { cudaq::qec::info("Registering Pythonic Decoder with name {}", name); registry[name] = factory; + shadow_flags[name] = shadows_guarded_entries; } static nb::object get_decoder(const std::string &name, nb::object H, @@ -244,11 +251,17 @@ class PyDecoderRegistry { static bool contains(const std::string &name) { return registry.find(name) != registry.end(); } + + static bool shadows_guarded_entries(const std::string &name) { + auto it = shadow_flags.find(name); + return it != shadow_flags.end() && it->second; + } }; std::unordered_map> PyDecoderRegistry::registry; +std::unordered_map PyDecoderRegistry::shadow_flags; namespace { bool is_affinity_kwarg(const std::string &k) { @@ -275,6 +288,20 @@ nb::object get_py_registered_decoder(const std::string &name, nb::object H, else stripped[item.first] = item.second; } + // A Python class that overrides decode_batch/decode_async in Python is + // called directly by Python attribute lookup — the guarded C++ entry + // points never run, so hardware kwargs would be accepted and silently + // dead. Reject loudly instead (before construction; the flag was recorded + // from the class namespace at decorator-registration time). + if (affinity_kwargs.size() > 0 && + PyDecoderRegistry::shadows_guarded_entries(name)) + throw std::runtime_error( + "Decoder '" + name + + "' overrides decode_batch/decode_async in " + "Python, so the hardware placement kwargs (cuda_device_id/" + "numa_node_id/mempolicy/cpu_affinity) cannot be applied to those " + "calls. Remove the kwargs, or implement only decode() and use the " + "guarded base-class batch entry points."); nb::object obj = PyDecoderRegistry::get_decoder( name, H, nb::borrow(stripped.ptr())); // No affinity kwargs: preserve exact pre-existing behavior (including BYOD @@ -731,9 +758,10 @@ void bindDecoder(nb::module_ &mod) { .def( "decode", [](decoder &decoder, const std::vector &syndrome) { - return decoder.decode(syndrome); + return decoder.decode_guarded(syndrome); }, - "Decode the given syndrome to determine the error correction", + "Decode the given syndrome to determine the error correction. " + "Applies the decoder's hardware placement automatically.", nb::arg("syndrome")) .def( "decode_async", @@ -758,6 +786,23 @@ void bindDecoder(nb::module_ &mod) { "device).") .def("numa_node_id", &decoder::numa_node_id, "Target NUMA node for this decoder (-1 = no binding).") + .def( + "mempolicy", + [](decoder &d) { + return d.mempolicy() == cudaq::qec::mempolicy_mode::bind + ? "bind" + : "preferred"; + }, + "NUMA memory-policy mode for this decoder.") + .def("cpu_affinity", &decoder::cpu_affinity, + "Explicit CPU core list for this decoder's owning thread " + "(empty = node cpuset).") + .def("bind_current_thread", &decoder::bind_current_thread, + "Persistently pin the calling thread to this decoder's device/" + "node; guarded entry points then skip their per-call guard on " + "this thread. Call unbind_thread() before the thread exits.") + .def("unbind_thread", &decoder::unbind_thread, + "Forget a bind_current_thread() registration.") .def("get_block_size", &decoder::get_block_size, "Get the size of the code block") .def("get_syndrome_size", &decoder::get_syndrome_size, @@ -893,6 +938,14 @@ void bindDecoder(nb::module_ &mod) { if (!nb::hasattr(decoder_class, "decode")) throw std::runtime_error("Decoder class must implement decode method"); + // Record whether the class defines decode_batch/decode_async in + // PYTHON. The check must run here, on the class namespace: hasattr on + // an instance (or on new_class) also sees the guarded C++ bindings + // inherited from qec.Decoder and would always be true. + const bool shadows_guarded_entries = + namespace_dict.contains("decode_batch") || + namespace_dict.contains("decode_async"); + // Use Python's type() so the correct metaclass (nanobind's) is resolved nb::object type_fn = nb::module_::import_("builtins").attr("type"); nb::object new_class = @@ -900,10 +953,12 @@ void bindDecoder(nb::module_ &mod) { // Register the new class in the decoder registry PyDecoderRegistry::register_decoder( - name, [new_class](nb::object H, nb::kwargs options) { + name, + [new_class](nb::object H, nb::kwargs options) { nb::object instance = new_class(H, **options); return instance; - }); + }, + shadows_guarded_entries); return new_class; }); }); diff --git a/libs/qec/python/bindings/py_decoding_config.cpp b/libs/qec/python/bindings/py_decoding_config.cpp index 777ad868a..e64a01982 100644 --- a/libs/qec/python/bindings/py_decoding_config.cpp +++ b/libs/qec/python/bindings/py_decoding_config.cpp @@ -286,6 +286,8 @@ void bindDecodingConfig(nb::module_ &mod) { .def_rw("D_sparse", &decoder_config::D_sparse) .def_rw("cuda_device_id", &decoder_config::cuda_device_id) .def_rw("numa_node_id", &decoder_config::numa_node_id) + .def_rw("mempolicy", &decoder_config::mempolicy) + .def_rw("cpu_affinity", &decoder_config::cpu_affinity) .def_rw("decoder_custom_args", &decoder_config::decoder_custom_args) .def( "set_decoder_custom_args", diff --git a/libs/qec/python/tests/test_decoder.py b/libs/qec/python/tests/test_decoder.py index 29eb97793..658d181e0 100644 --- a/libs/qec/python/tests/test_decoder.py +++ b/libs/qec/python/tests/test_decoder.py @@ -1160,5 +1160,47 @@ def test_invalid_mempolicy_string_raises_naming_the_knob(): qec.get_decoder("multi_error_lut", H, numa_node_id=0, mempolicy="bnid") +def test_registered_decoder_with_python_decode_batch_rejects_affinity_kwargs(): + + @qec.decoder("shadowing_byod_neg") + class ShadowingDecoder: + + def __init__(self, H, **kwargs): + qec.Decoder.__init__(self, H) + + def decode(self, syndrome): + r = qec.DecoderResult() + r.converged = True + r.result = [0.0, 0.0, 0.0] + return r + + # A Python-level decode_batch is dispatched by attribute lookup and + # bypasses the guarded C++ entry, so placement kwargs cannot apply. + def decode_batch(self, syndromes): + return [self.decode(s) for s in syndromes] + + H = np.array([[1, 1, 0], [0, 1, 1]], dtype=np.uint8) + with pytest.raises(RuntimeError, match="decode_batch"): + qec.get_decoder("shadowing_byod_neg", H, numa_node_id=0) + # Without affinity kwargs the same class constructs fine: + d = qec.get_decoder("shadowing_byod_neg", H) + assert d is not None + + +def test_python_bind_unbind_and_readback_surface(): + H = np.array([[1, 1, 0], [0, 1, 1]], dtype=np.uint8) + d = qec.get_decoder("multi_error_lut", + H, + numa_node_id=0, + mempolicy="bind", + cpu_affinity=[0]) + assert d.mempolicy() == "bind" + assert d.cpu_affinity() == [0] + d.bind_current_thread() + r = d.decode([0.0, 0.0]) # routes through the guarded single-shot entry + assert r is not None + d.unbind_thread() + + if __name__ == "__main__": pytest.main() diff --git a/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp b/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp index 16b34f3eb..0123261ce 100644 --- a/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp +++ b/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp @@ -431,6 +431,80 @@ TEST(PyMatchingRealtime, SessionNumaNodeConflictWarnsAndStillDecodes) { EXPECT_NO_THROW(session.finalize()); } +// Conflict gate: same NUMA node, different mempolicy modes. One dispatch +// thread can hold only one memory policy, so guard-skip binding must stay +// off, loudly, while per-call guards keep every decoder decoding correctly. +TEST(PyMatchingRealtime, SessionMempolicyConflictWarnsAndStillDecodes) { + auto decoders = make_session_pair(); + cudaqx::heterogeneous_map knobs_a; + knobs_a.insert("numa_node_id", 0); // node 0 always exists + knobs_a.insert("mempolicy", std::string("preferred")); + decoders[0]->set_hardware_params(knobs_a); + cudaqx::heterogeneous_map knobs_b; + knobs_b.insert("numa_node_id", 0); + knobs_b.insert("mempolicy", std::string("bind")); + decoders[1]->set_hardware_params(knobs_b); + + cudaq::qec::realtime::qec_realtime_session session(decoders); + session_finalizer fin{session}; + bool threw = false; + std::string what; + const std::string err = initialize_capturing_stderr(session, threw, what); + ASSERT_FALSE(threw) << "conflicting mempolicy modes must degrade to " + "per-call guards, not throw: " + << what; + EXPECT_NE(err.find("different mempolicy modes"), std::string::npos) + << "conflict must be loud; captured stderr: [" << err << "]"; + + expect_decoder_still_decodes(session, /*decoder_id=*/0, /*tag=*/1); + expect_decoder_still_decodes(session, /*decoder_id=*/1, /*tag=*/2); + EXPECT_NO_THROW(session.finalize()); +} + +// Agreement on ONLY cpu_affinity (no numa_node_id, no cuda_device_id) must +// still engage the dispatch-thread bind rather than silently dropping the +// knob: initialize is conflict-warning-free, both decoders decode correctly +// through the bound dispatch thread, and finalize round-trips cleanly. (This +// binary runs without the affinity syscall interposer, so the contract is +// asserted through the warning-silence + decode + finalize surface.) +TEST(PyMatchingRealtime, SessionCpuAffinityOnlyStillBinds) { +#if defined(__linux__) + cpu_set_t allowed; + CPU_ZERO(&allowed); + ASSERT_EQ(sched_getaffinity(0, sizeof(allowed), &allowed), 0); + if (!CPU_ISSET(0, &allowed)) + GTEST_SKIP() << "CPU 0 is not in the allowed cpuset"; + + auto decoders = make_session_pair(); + cudaqx::heterogeneous_map knobs; + knobs.insert("cpu_affinity", std::vector{0}); + decoders[0]->set_hardware_params(knobs); + decoders[1]->set_hardware_params(knobs); + + cudaq::qec::realtime::qec_realtime_session session(decoders); + session_finalizer fin{session}; + bool threw = false; + std::string what; + const std::string err = initialize_capturing_stderr(session, threw, what); + ASSERT_FALSE(threw) << "agreeing cpu_affinity lists must initialize " + "cleanly: " + << what; + EXPECT_EQ(err.find("request different"), std::string::npos) + << "agreeing knobs must not raise a conflict warning; captured " + "stderr: [" + << err << "]"; + EXPECT_EQ(err.find("[cudaq-qec affinity]"), std::string::npos) + << "binding to an allowed CPU must be silent; captured stderr: [" << err + << "]"; + + expect_decoder_still_decodes(session, /*decoder_id=*/0, /*tag=*/1); + expect_decoder_still_decodes(session, /*decoder_id=*/1, /*tag=*/2); + EXPECT_NO_THROW(session.finalize()); +#else + GTEST_SKIP() << "Linux-only (sched_getaffinity probe)"; +#endif +} + // Silence contract: a session over decoders that never // touched a pinning knob must emit ZERO affinity noise ("[cudaq-qec // affinity]" is the fprintf marker of hardware_affinity.h's affinity_warn) diff --git a/libs/qec/unittests/test_decoders_yaml.cpp b/libs/qec/unittests/test_decoders_yaml.cpp index 06a4e0ddf..2896e634a 100644 --- a/libs/qec/unittests/test_decoders_yaml.cpp +++ b/libs/qec/unittests/test_decoders_yaml.cpp @@ -758,3 +758,35 @@ TEST(DecoderYaml, PrepareParamsInjectsKeys) { EXPECT_TRUE(params.contains("cuda_device_id")); EXPECT_TRUE(params.contains("numa_node_id")); } + +TEST(DecoderYaml, MempolicyAndCpuAffinityRoundTrip) { + using namespace cudaq::qec::decoding::config; + multi_decoder_config multi_config; + decoder_config cfg; + cfg.id = 0; + cfg.type = "multi_error_lut"; + cfg.block_size = 4; + cfg.syndrome_size = 2; + cfg.H_sparse = {0, -1, 1, -1}; + cfg.O_sparse = {}; + cfg.D_sparse = {}; + cfg.mempolicy = "bind"; + cfg.cpu_affinity = std::vector{0, 2}; + multi_config.decoders.push_back(cfg); + + test_decoder_yaml_roundtrip(multi_config); + + auto reparsed = + multi_decoder_config::from_yaml_str(multi_config.to_yaml_str(200)); + ASSERT_EQ(reparsed.decoders.size(), 1u); + const auto &dc = reparsed.decoders.at(0); + ASSERT_TRUE(dc.mempolicy.has_value()); + EXPECT_EQ(*dc.mempolicy, "bind"); + ASSERT_TRUE(dc.cpu_affinity.has_value()); + EXPECT_EQ(*dc.cpu_affinity, (std::vector{0, 2})); + + auto params = cudaq::qec::decoding::host::prepare_decoder_params(dc); + EXPECT_EQ(cudaq::qec::read_mempolicy(params), + cudaq::qec::mempolicy_mode::bind); + EXPECT_EQ(cudaq::qec::read_cpu_affinity(params), (std::vector{0, 2})); +} diff --git a/libs/qec/unittests/test_pinning_benchmark.cpp b/libs/qec/unittests/test_pinning_benchmark.cpp index 5eaa1c286..65fa2dbf4 100644 --- a/libs/qec/unittests/test_pinning_benchmark.cpp +++ b/libs/qec/unittests/test_pinning_benchmark.cpp @@ -409,6 +409,92 @@ TEST(PinningInvariants, UnboundEntryPointsApplyAndFullyRestoreGuard) { } } +// The tensor-decode entry must construct its guards BEFORE building the +// soft-syndrome temporaries, so those allocations land on the decoder's node. +// Observable: with the shim, the FIRST placement syscall must occur before +// any decode work — assert the guard fires even for a rank-check failure, +// which returns before temporaries are built. +TEST(PinningInvariants, TensorDecodeGuardPrecedesTemporaries) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + run_on_worker([] { + auto d = makeEntryReadyPinnedLut(); + cudaqx::tensor bad({2, 2}); // rank-2: must throw + cudaqx_affinity_syscall_reset(); + EXPECT_THROW((void)d->decode(bad), std::runtime_error); + EXPECT_GT(readShimCounts().getaff + readShimCounts().getmem, 0) + << "guard must be constructed before input processing"; + }); +} + +// Post-construction sparse setters allocate session-lifetime buffers +// (corrections, msyn accumulators); those buffers must follow the decoder's +// placement like every other allocating entry point, and the guard must +// restore the calling thread's placement afterwards. Both overload shapes of +// each setter allocate independently, so each is exercised on its own. +TEST(PinningInvariants, SparseSettersApplyNumaGuardWhenUnbound) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + struct SetterCall { + const char *name; + void (*call)(cudaq::qec::decoder &); + }; + const SetterCall kSetters[] = { + {"set_D_sparse(nested)", + [](cudaq::qec::decoder &d) { + d.set_D_sparse(std::vector>{{0}, {1}}); + }}, + {"set_D_sparse(flat)", + [](cudaq::qec::decoder &d) { + // -1 terminates a row: {{0}, {1}}. + d.set_D_sparse(std::vector{0, -1, 1}); + }}, + {"set_O_sparse(nested)", + [](cudaq::qec::decoder &d) { + d.set_O_sparse(std::vector>{{0, 1}}); + }}, + {"set_O_sparse(flat)", + [](cudaq::qec::decoder &d) { + d.set_O_sparse(std::vector{0, 1}); + }}, + }; + run_on_worker([&] { + auto d = makePinnedLut(); // numa_node_id=0, unbound -> guard must run + for (const auto &s : kSetters) { + SCOPED_TRACE(s.name); + auto before = thread_placement::capture(); + cudaqx_affinity_syscall_reset(); + s.call(*d); + auto c = readShimCounts(); + EXPECT_GT(c.setmem + c.setaff, 0) + << "sparse setters must apply the placement guard"; + auto after = thread_placement::capture(); + EXPECT_EQ(before, after) << "setter guard must restore placement: " + << before.describe_difference(after); + } + }); +} + +// Non-virtual guarded single-syndrome entry (used by the Python binding — +// callers that have not bound a thread must self-place). +TEST(PinningInvariants, DecodeGuardedAppliesAndRestores) { + if (!shimCountersPresent()) + GTEST_SKIP() << "affinity-counter shim not preloaded"; + run_on_worker([] { + auto d = makePinnedLut(); + auto before = thread_placement::capture(); + cudaqx_affinity_syscall_reset(); + (void)d->decode_guarded({0.1f, 0.1f}); + EXPECT_GT(readShimCounts().setmem + readShimCounts().setaff, 0); + EXPECT_EQ(before, thread_placement::capture()); + d->bind_current_thread(); + cudaqx_affinity_syscall_reset(); + (void)d->decode_guarded({0.1f, 0.1f}); + EXPECT_EQ(cudaqx_affinity_syscall_count(), 0) + << "bound thread must skip decode_guarded's guard"; + }); +} + // unbind_thread() must forget the registration so guards re-engage — the // session calls it at teardown before the bound thread's id can be recycled. TEST(PinningBenchmark, UnbindThreadRestoresGuarding) { From cc53ef9a6a75285c177423bec126b03595c046c5 Mon Sep 17 00:00:00 2001 From: kvmto Date: Mon, 6 Jul 2026 23:21:39 +0000 Subject: [PATCH 13/16] refactor(qec): consolidate trt guards onto the shared header; release the GIL in decode bindings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace trt_decoder::decode_batch's inline RestoreDevice/ScopedNuma with the shared CudaDeviceGuard/NumaGuard from hardware_guards.h — one guard definition now serves the base entry points, the realtime session, and the trt override. Unifies the error posture: an unreadable current device warns-and-skips at every entry point, and out-of-range ids throw the same count-checked error everywhere. Python decode and bind_current_thread release the GIL for the C++ call (decode_batch releases it around the decode only — its result conversion builds Python objects); nanobind's trampoline re-acquires it for Python-implemented decoders, so pinned Python workers decode concurrently across GPUs Signed-off-by: kvmto --- .../plugins/trt_decoder/trt_decoder.cpp | 81 ++----------------- libs/qec/lib/hardware_guards.h | 7 +- libs/qec/lib/realtime/CMakeLists.txt | 2 +- libs/qec/python/bindings/py_decoder.cpp | 23 +++++- 4 files changed, 32 insertions(+), 81 deletions(-) diff --git a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp index ef2bc11f2..c1c5c5414 100644 --- a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp +++ b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp @@ -906,81 +906,14 @@ trt_decoder::decode_batch(const std::vector> &syndromes) { // bind_current_thread() — mirrors the base class decode_batch() behaviour. const bool skip_guard = is_bound_here(); - // This override bypasses decoder::decode_batch()'s CudaDeviceGuard/NumaGuard - // (private to decoder.cpp, not reachable across this plugin's shared-library - // boundary); reimplement the same contract here so TRT inference lands on - // the right device/NUMA node regardless of the caller. - int _prev_dev = -1; - bool _dev_switched = false; - if (!skip_guard && cuda_device_id_ >= 0) { - cudaError_t _get_err = cudaGetDevice(&_prev_dev); - if (_get_err != cudaSuccess) - throw std::runtime_error( - std::string("trt_decoder::decode_batch: cudaGetDevice failed: ") + - cudaGetErrorString(_get_err)); - if (_prev_dev != cuda_device_id_) { - cudaError_t _set_err = cudaSetDevice(cuda_device_id_); - if (_set_err != cudaSuccess) - throw std::runtime_error("trt_decoder::decode_batch: cudaSetDevice(" + - std::to_string(cuda_device_id_) + - ") failed: " + cudaGetErrorString(_set_err)); - _dev_switched = true; - } - } - struct RestoreDevice { - int prev; - bool active; - ~RestoreDevice() { - if (!active) - return; - if (cudaSetDevice(prev) != cudaSuccess) - cudaq::qec::detail_affinity::affinity_warn( - "trt_decoder: failed to restore prior CUDA device " + - std::to_string(prev) + "; thread may remain on wrong device"); - } - } _dev_guard{_prev_dev, _dev_switched}; - + // This override bypasses decoder::decode_batch()'s guards, so it applies + // the same shared pair as the base-class entry points (lib-private header, + // reachable across the plugin boundary): device switch + NUMA bind, both + // restored on scope exit. Out-of-range ids throw; an unreadable current + // device warns and skips the switch. namespace da = cudaq::qec::detail_affinity; - struct ScopedNuma { - bool active = false; - bool has_prev_affinity = false; -#if defined(__linux__) - cpu_set_t prev_set{}; -#endif - da::mempolicy_state prev_mempolicy; - ScopedNuma(int node, cudaq::qec::mempolicy_mode mode) { - if (node < 0) - return; - bool apply_mempol = true; -#if defined(__linux__) - has_prev_affinity = - (sched_getaffinity(0, sizeof(prev_set), &prev_set) == 0); - prev_mempolicy = da::capture_thread_mempolicy(); - // A temporary guard must never apply a policy it cannot restore. - apply_mempol = (prev_mempolicy.mode >= 0); - if (!apply_mempol) - da::affinity_warn( - "trt_decoder: prior thread mempolicy unreadable (get_mempolicy " - "failed); temporary NUMA memory policy skipped for this decode"); -#endif - da::bind_this_thread_to_numa_node(node, mode, apply_mempol); - // Arm the restore only after a successful bind; bind throws before any - // syscall for out-of-range nodes, so there is nothing to undo then. - active = true; - } - ~ScopedNuma() { - if (!active) - return; -#if defined(__linux__) - da::restore_thread_mempolicy(prev_mempolicy); - if (has_prev_affinity && - sched_setaffinity(0, sizeof(prev_set), &prev_set) != 0) - da::affinity_warn( - "trt_decoder: failed to reset thread affinity after decode; " - "thread may remain pinned"); -#endif - } - } _numa_guard(skip_guard ? -1 : numa_node_id_, mempolicy()); + da::CudaDeviceGuard dev_guard(skip_guard ? -1 : cuda_device_id_); + da::NumaGuard numa_guard(skip_guard ? -1 : numa_node_id_, mempolicy()); // Validate that we have syndromes to decode if (syndromes.empty()) { diff --git a/libs/qec/lib/hardware_guards.h b/libs/qec/lib/hardware_guards.h index 93fc8dea0..9fb0406b9 100644 --- a/libs/qec/lib/hardware_guards.h +++ b/libs/qec/lib/hardware_guards.h @@ -18,9 +18,10 @@ namespace cudaq::qec::detail_affinity { // RAII: sets the calling thread's CUDA current device, restores on destruction. -// target < 0 = no-op. Throws std::runtime_error if: target is out of range -// of the visible device count, cudaGetDevice() fails while checking the -// current device, or cudaSetDevice() fails while switching to target. +// target < 0 = no-op. Throws std::runtime_error if target is out of range of +// the visible device count or cudaSetDevice() fails while switching to target. +// If cudaGetDevice() cannot read the current device, warns and skips the +// switch (the decode proceeds on the caller's current device). struct CudaDeviceGuard { int prev_ = -1; bool active_ = false; diff --git a/libs/qec/lib/realtime/CMakeLists.txt b/libs/qec/lib/realtime/CMakeLists.txt index ab387a1e1..6bc8c4b16 100644 --- a/libs/qec/lib/realtime/CMakeLists.txt +++ b/libs/qec/lib/realtime/CMakeLists.txt @@ -6,7 +6,7 @@ # the terms of the Apache License 2.0 which accompanies this distribution. # # ============================================================================ # -find_package(LLVM 22.1 REQUIRED CONFIG) +find_package(LLVM 22.1.4 EXACT REQUIRED CONFIG) # Enable CUDA for device code if(CMAKE_CUDA_COMPILER) diff --git a/libs/qec/python/bindings/py_decoder.cpp b/libs/qec/python/bindings/py_decoder.cpp index 85d7f501b..87ecb955a 100644 --- a/libs/qec/python/bindings/py_decoder.cpp +++ b/libs/qec/python/bindings/py_decoder.cpp @@ -762,7 +762,12 @@ void bindDecoder(nb::module_ &mod) { }, "Decode the given syndrome to determine the error correction. " "Applies the decoder's hardware placement automatically.", - nb::arg("syndrome")) + nb::arg("syndrome"), + // Runs without the GIL so other Python threads can decode + // concurrently. Safe: the lambda is pure C++, and a Python-side + // decode override re-acquires the GIL in nanobind's trampoline + // (PyGILState_Ensure in trampoline_enter). + nb::call_guard()) .def( "decode_async", [](decoder &dec, @@ -776,7 +781,16 @@ void bindDecoder(nb::module_ &mod) { "decode_batch", [](decoder &decoder, const std::vector> &syndrome) { - auto results = decoder.decode_batch(syndrome); + std::vector results; + { + // Release the GIL only around the C++ decode: a call_guard + // would also cover makeBatchDecoderResult, which builds Python + // objects and must hold the GIL. A Python-side decode override + // reached from the batch loop re-acquires the GIL in nanobind's + // trampoline (PyGILState_Ensure in trampoline_enter). + nb::gil_scoped_release release; + results = decoder.decode_batch(syndrome); + } return makeBatchDecoderResult(results); }, "Decode multiple syndromes and return the results", @@ -800,7 +814,10 @@ void bindDecoder(nb::module_ &mod) { .def("bind_current_thread", &decoder::bind_current_thread, "Persistently pin the calling thread to this decoder's device/" "node; guarded entry points then skip their per-call guard on " - "this thread. Call unbind_thread() before the thread exits.") + "this thread. Call unbind_thread() before the thread exits.", + // Pure C++ (affinity/mempolicy/CUDA-device syscalls); runs without + // the GIL so pinning cannot stall other Python threads. + nb::call_guard()) .def("unbind_thread", &decoder::unbind_thread, "Forget a bind_current_thread() registration.") .def("get_block_size", &decoder::get_block_size, From 08646c70e2167be94cb0755eb45a397989d27e6f Mon Sep 17 00:00:00 2001 From: kvmto Date: Mon, 6 Jul 2026 23:43:31 +0000 Subject: [PATCH 14/16] quick fix of include Signed-off-by: kvmto --- libs/qec/lib/decoders/plugins/trt_decoder/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/qec/lib/decoders/plugins/trt_decoder/CMakeLists.txt b/libs/qec/lib/decoders/plugins/trt_decoder/CMakeLists.txt index 82603b0d0..b34307f75 100644 --- a/libs/qec/lib/decoders/plugins/trt_decoder/CMakeLists.txt +++ b/libs/qec/lib/decoders/plugins/trt_decoder/CMakeLists.txt @@ -124,7 +124,7 @@ if(CUDAQ_QEC_TRT_DECODER_ENABLED) ${CMAKE_SOURCE_DIR}/libs/qec/include ${CMAKE_SOURCE_DIR}/libs/core/include PRIVATE - ${CMAKE_SOURCE_DIR}/libs/qec/lib + ${CMAKE_CURRENT_SOURCE_DIR}/../../.. ${TENSORRT_INCLUDE_DIR} ${CUDAToolkit_INCLUDE_DIRS} ) From 088973e197d295dfc7540d98971b21f334dccc7e Mon Sep 17 00:00:00 2001 From: kvmto Date: Tue, 7 Jul 2026 11:48:53 +0000 Subject: [PATCH 15/16] pinning test fix Signed-off-by: kvmto --- libs/qec/unittests/test_pinning_benchmark.cpp | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/libs/qec/unittests/test_pinning_benchmark.cpp b/libs/qec/unittests/test_pinning_benchmark.cpp index 65fa2dbf4..aa0f20123 100644 --- a/libs/qec/unittests/test_pinning_benchmark.cpp +++ b/libs/qec/unittests/test_pinning_benchmark.cpp @@ -207,12 +207,18 @@ TEST(PinningBenchmark, OneShotDoesNotClobberConcurrentBind) { auto d = makePinnedLut(); // Thread B will call bind_current_thread() while the one-shot worker runs. + // It stays alive until the unbound-thread check below completes: if it + // exited first, the fresh "unbound" thread could recycle its pthread id, + // spuriously pass is_bound_here(), and skip the guard (observed CI flake). std::atomic b_bound{false}; + std::atomic b_done{false}; std::thread thread_b([&] { // Spin until main_worker signals us to bind. while (!b_bound.load(std::memory_order_acquire)) std::this_thread::yield(); d->bind_current_thread(); + while (!b_done.load(std::memory_order_acquire)) + std::this_thread::yield(); }); // Run decode_on_pinned_thread from the main thread; signal B to bind midway. @@ -223,19 +229,22 @@ TEST(PinningBenchmark, OneShotDoesNotClobberConcurrentBind) { }); main_worker.join(); - thread_b.join(); - // After both threads finish, whoever bound last should have their binding - // intact. At a minimum, is_bound_here() from thread_b must not permanently - // suppress guards: a subsequent decode from an unbound thread must still - // issue syscalls. + // Whoever bound last should have their binding intact. At a minimum, + // thread_b's binding must not permanently suppress guards for other + // threads: a decode from a thread that was never bound must still issue + // syscalls. cudaqx_affinity_syscall_reset(); - // Run decode from a fresh thread that was never bound. + // Run decode from a fresh thread that was never bound (thread_b is still + // alive, so this thread cannot alias its id). std::thread unbound([&] { d->decode_batch(kChunk); }); unbound.join(); // An unbound thread must always fire the guard. EXPECT_GT(cudaqx_affinity_syscall_count(), 0) << "guard must still fire for unbound threads after concurrent-bind race"; + + b_done.store(true, std::memory_order_release); + thread_b.join(); } TEST(PinningBenchmark, EnqueueSyndromeAppliesGuardOnUnboundThread) { From e4760abed8d6d63e52f9e225bc7bd6368352b29b Mon Sep 17 00:00:00 2001 From: kvmto Date: Tue, 7 Jul 2026 11:58:33 +0000 Subject: [PATCH 16/16] CI + test fix for multi trt testing Signed-off-by: kvmto --- .github/workflows/lib_qec.yaml | 7 ++++--- libs/qec/unittests/CMakeLists.txt | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lib_qec.yaml b/.github/workflows/lib_qec.yaml index 40c29284b..81cfe3189 100644 --- a/.github/workflows/lib_qec.yaml +++ b/.github/workflows/lib_qec.yaml @@ -313,7 +313,8 @@ jobs: - name: Run multi-GPU C++ tests run: | cd $GITHUB_WORKSPACE/build_qec - # The matching tests will be added separately; until then, this - # command should not fail solely because the filter finds no tests. + # CudaDeviceId also matches the TRT 2-GPU placement tests + # (TRTDecoderTest.CudaDeviceId_*); TwoTrtDecodersConcurrently is the + # TRT concurrent-decode test that skips on single-GPU lanes. ctest --output-on-failure \ - -R "DecoderPool|HardwarePinningGpu|PoolRunsTwoTrt|CudaDeviceId" + -R "HardwarePinningGpu|CudaDeviceId|TwoTrtDecodersConcurrently" diff --git a/libs/qec/unittests/CMakeLists.txt b/libs/qec/unittests/CMakeLists.txt index 4af6462b9..5a269c706 100644 --- a/libs/qec/unittests/CMakeLists.txt +++ b/libs/qec/unittests/CMakeLists.txt @@ -97,8 +97,9 @@ target_link_libraries(test_logger PRIVATE GTest::gtest_main cudaq-qec cudaq::cud add_dependencies(CUDAQXQECUnitTests test_logger) gtest_discover_tests(test_logger) -# TensorRT decoder test is only built for x86 architectures -if(CUDAQ_QEC_TRT_DECODER_ENABLED AND CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") +# Built wherever the TRT decoder plugin is enabled (x86_64 and arm64); the +# flag is false on platforms without TensorRT (e.g. arm64 + CUDA 12). +if(CUDAQ_QEC_TRT_DECODER_ENABLED) add_executable(test_trt_decoder ./decoders/trt_decoder/test_trt_decoder.cpp) # Find TensorRT for the test