diff --git a/.cudaq_version b/.cudaq_version index 6357bcab6..8d461eb77 100644 --- a/.cudaq_version +++ b/.cudaq_version @@ -1,6 +1,6 @@ { "cudaq": { - "repository": "NVIDIA/cuda-quantum", - "ref": "5f0ee5be3bf5cb33afb544d9f01af77b1d50b26e" + "repository": "anjbur/cuda-quantum", + "ref": "d16bf937633078e81e558a4c4972fa811153eb42" } } diff --git a/.github/actions/build-lib/build_all.sh b/.github/actions/build-lib/build_all.sh index e772a0a8d..78d5961cf 100755 --- a/.github/actions/build-lib/build_all.sh +++ b/.github/actions/build-lib/build_all.sh @@ -10,10 +10,6 @@ _rt_flag="" if [ -n "$CUDAQ_REALTIME_ROOT" ]; then _rt_flag="-DCUDAQ_REALTIME_ROOT=$CUDAQ_REALTIME_ROOT" fi -_prop_archive_flag="" -if [ -n "$CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE" ]; then - _prop_archive_flag="-DCUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE=$CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE" -fi cmake -S . -B "$build_dir" \ -DCMAKE_BUILD_TYPE=Release \ @@ -26,7 +22,6 @@ cmake -S . -B "$build_dir" \ -DCUDAQX_INCLUDE_TESTS=ON \ -DCUDAQX_BINDINGS_PYTHON=ON \ -DCMAKE_INSTALL_PREFIX="$install_prefix" \ - $_rt_flag \ - $_prop_archive_flag + $_rt_flag cmake --build "$build_dir" --target install -j 4 diff --git a/.github/actions/build-lib/build_qec.sh b/.github/actions/build-lib/build_qec.sh index e53e08300..e73742c2e 100755 --- a/.github/actions/build-lib/build_qec.sh +++ b/.github/actions/build-lib/build_qec.sh @@ -31,6 +31,10 @@ if [ -z "$CUDAQ_REALTIME_ROOT" ]; then CUDA_MAJOR_VERSION=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\).*$/\1/p') apt-get update && apt-get install -y --no-install-recommends \ ninja-build curl pkg-config + # HSB -> find_package(holoscan) -> rapids_logger requires cmake >= 3.30.4; + # the CI container ships cmake 3.28. + pip install 'cmake<4' + export PATH="$(python3 -c 'import cmake,os;print(os.path.join(os.path.dirname(cmake.__file__),"data","bin"))'):$PATH" # Add DOCA repo and install only the GPUNetIO dev package (not doca-all) DOCA_ARCH=$(uname -m) @@ -114,11 +118,6 @@ fi HSB_ROOT=/tmp/holoscan-sensor-bridge HSB_BUILD=${HSB_ROOT}/build -_prop_archive_flag="" -if [ -n "$CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE" ]; then - _prop_archive_flag="-DCUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE=$CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE" -fi - cmake -S libs/qec -B "$build_dir" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER=gcc-12 \ @@ -132,7 +131,6 @@ cmake -S libs/qec -B "$build_dir" \ -DCUDAQ_REALTIME_ROOT=$CUDAQ_REALTIME_ROOT \ -DCUDAQX_QEC_ENABLE_HOLOLINK_TOOLS=ON \ -DHOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR=$HSB_ROOT \ - -DHOLOSCAN_SENSOR_BRIDGE_BUILD_DIR=$HSB_BUILD \ - $_prop_archive_flag + -DHOLOSCAN_SENSOR_BRIDGE_BUILD_DIR=$HSB_BUILD cmake --build "$build_dir" --target install -j 4 diff --git a/.github/actions/get-cudaq-wheels/action.yaml b/.github/actions/get-cudaq-wheels/action.yaml index 268ff0e32..add9b71fc 100644 --- a/.github/actions/get-cudaq-wheels/action.yaml +++ b/.github/actions/get-cudaq-wheels/action.yaml @@ -125,7 +125,7 @@ runs: context: cudaq file: ./cudaq/docker/release/cudaq.wheel.Dockerfile build-args: | - base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ inputs.platform }}-cu${{ inputs.cuda_version }}-gcc12-main + base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ inputs.platform }}-cu${{ inputs.cuda_version }}-gcc12-cudaqx-0.15.0 release_version=0.14.99 python_version=3.11 outputs: /cudaq-wheels @@ -137,7 +137,7 @@ runs: context: cudaq file: ./cudaq/docker/release/cudaq.wheel.Dockerfile build-args: | - base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ inputs.platform }}-cu${{ inputs.cuda_version }}-gcc12-main + base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ inputs.platform }}-cu${{ inputs.cuda_version }}-gcc12-cudaqx-0.15.0 release_version=0.14.99 python_version=3.12 outputs: /cudaq-wheels @@ -149,7 +149,7 @@ runs: context: cudaq file: ./cudaq/docker/release/cudaq.wheel.Dockerfile build-args: | - base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ inputs.platform }}-cu${{ inputs.cuda_version }}-gcc12-main + base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ inputs.platform }}-cu${{ inputs.cuda_version }}-gcc12-cudaqx-0.15.0 release_version=0.14.99 python_version=3.13 outputs: /cudaq-wheels diff --git a/.github/workflows/all_libs.yaml b/.github/workflows/all_libs.yaml index 8605b2c13..43e79ecee 100644 --- a/.github/workflows/all_libs.yaml +++ b/.github/workflows/all_libs.yaml @@ -15,6 +15,10 @@ on: type: string default: '' required: false + cudaq_dev_images: + type: string + default: '{"amd64-cu12.6":"ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu12.6-gcc12-main","amd64-cu13.0":"ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu13.0-gcc12-main","arm64-cu12.6":"ghcr.io/nvidia/cuda-quantum-devcontainer:arm64-cu12.6-gcc12-main","arm64-cu13.0":"ghcr.io/nvidia/cuda-quantum-devcontainer:arm64-cu13.0-gcc12-main"}' + required: false jobs: build-and-test: @@ -25,7 +29,7 @@ jobs: platform: ['amd64', 'arm64'] cuda_version: ['12.6', '13.0'] runs-on: ${{ startsWith(github.repository, 'NVIDIA/cudaqx') && format('linux-{0}-cpu8', matrix.platform) || 'ubuntu-latest' }} - container: ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main + container: ${{ fromJSON(inputs.cudaq_dev_images)[format('{0}-cu{1}', matrix.platform, matrix.cuda_version)] }} permissions: actions: write contents: read diff --git a/.github/workflows/all_libs_release.yaml b/.github/workflows/all_libs_release.yaml index 05128884c..70f1b9368 100644 --- a/.github/workflows/all_libs_release.yaml +++ b/.github/workflows/all_libs_release.yaml @@ -85,10 +85,7 @@ jobs: # Extract the decoder that needs to be embedded in the release mkdir -p tmp tar -C tmp -xzvf nv-qldpc-decoder-${{ matrix.runner.arch }}_ubuntu24.04_cuda${{ matrix.cuda_version }}_release.tar.gz - test -f tmp/libcudaq-qec-nv-qldpc-decoder.so - test -f tmp/libcudaq-qec-realtime-cudevice-proprietary.a echo "QEC_EXTERNAL_DECODERS=$(pwd)/tmp/libcudaq-qec-nv-qldpc-decoder.so" >> $GITHUB_ENV - echo "CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE=$(pwd)/tmp/libcudaq-qec-realtime-cudevice-proprietary.a" >> $GITHUB_ENV fi shell: bash diff --git a/.github/workflows/build_dev.yaml b/.github/workflows/build_dev.yaml index 1b0ca5031..c18d0db6e 100644 --- a/.github/workflows/build_dev.yaml +++ b/.github/workflows/build_dev.yaml @@ -49,7 +49,7 @@ jobs: id: get-cudaq-version-short run: | shortref=$(echo "${{ steps.get-cudaq-version.outputs.ref }}" | head -c8) - commit_date=$(curl -s "https://api.github.com/repos/NVIDIA/cuda-quantum/commits/${{ steps.get-cudaq-version.outputs.ref }}" | jq -r '.commit.committer.date' | cut -dT -f1) + commit_date=$(curl -s "https://api.github.com/repos/${{ steps.get-cudaq-version.outputs.repo }}/commits/${{ steps.get-cudaq-version.outputs.ref }}" | jq -r '.commit.committer.date' | cut -dT -f1) echo "shortref=$shortref" >> $GITHUB_OUTPUT echo "commit_date=$commit_date" >> $GITHUB_OUTPUT @@ -86,8 +86,14 @@ jobs: if [ -n "$other_tag" ]; then TAGS+=" -t ghcr.io/nvidia/cudaqx-dev:${other_tag}-${{ matrix.platform }}-cu${{ matrix.cuda_version }}" fi + image_key="${{ matrix.platform }}-cu${{ matrix.cuda_version }}" + base_image=$(jq -r --arg key "$image_key" '.[$key] // empty' .github/workflows/cudaq-dev-images.json) + if [ -z "$base_image" ]; then + echo "No CUDA-Q devcontainer image configured for $image_key" >&2 + exit 1 + fi docker build $TAGS -f docker/build_env/cudaqx.dev.Dockerfile . \ - --build-arg base_image=ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main \ + --build-arg base_image="$base_image" \ --build-arg cuda_version=${{ matrix.cuda_version }} docker push -a ghcr.io/nvidia/cudaqx-dev shell: bash --noprofile --norc -euo pipefail {0} @@ -159,7 +165,7 @@ jobs: context: cudaq file: cudaq/docker/release/cudaq.wheel.Dockerfile build-args: | - base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main + base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-cudaqx-0.15.0 release_version=0.14.99 python_version=${{ matrix.python }} cuda_version=${{ matrix.cuda_version }} @@ -190,7 +196,7 @@ jobs: if [ -n "$other_tag" ]; then TAGS+=" -t ghcr.io/nvidia/cudaqx-dev:${other_tag}-py${{ matrix.python }}-${{ matrix.platform }}-cu${{ matrix.cuda_version }}" fi - BUILDARGS="--build-arg base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main" + BUILDARGS="--build-arg base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-cudaqx-0.15.0" BUILDARGS+=" --build-arg python_version=${{ matrix.python }}" BUILDARGS+=" --build-arg cuda_version=${{ matrix.cuda_version }}" BUILDARGS+=" --ulimit nofile=1048576:1048576" diff --git a/.github/workflows/build_wheels.yaml b/.github/workflows/build_wheels.yaml index 14d48ba09..d4ae747ac 100644 --- a/.github/workflows/build_wheels.yaml +++ b/.github/workflows/build_wheels.yaml @@ -99,7 +99,7 @@ jobs: # FIXME: there is no guarantee that this CUDA-Q image aligns with the CUDA-Q # commit that we are trying to align with. container: - image: ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main + image: ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-cudaqx-0.15.0 options: --ulimit nofile=1048576:1048576 permissions: actions: write diff --git a/.github/workflows/cudaq-dev-images.json b/.github/workflows/cudaq-dev-images.json new file mode 100644 index 000000000..eb2d0af9d --- /dev/null +++ b/.github/workflows/cudaq-dev-images.json @@ -0,0 +1,6 @@ +{ + "amd64-cu12.6": "ghcr.io/nvidia/cuda-quantum-devcontainer@sha256:46d0b72f85b083b8fb20af243eeca1308fbdee4fa2e87d1166f029a38d0d72e7", + "amd64-cu13.0": "ghcr.io/nvidia/cuda-quantum-devcontainer@sha256:da93c244cfe54d154b486e8e79ec94f2da6103ad0345246d8779286a72b60d6a", + "arm64-cu12.6": "ghcr.io/nvidia/cuda-quantum-devcontainer@sha256:c8928ddfc5d9acea2dcd9cb5fdfd11636b0aac73b1717c915d82ad972759261a", + "arm64-cu13.0": "ghcr.io/nvidia/cuda-quantum-devcontainer@sha256:50ebd24617f930e5e8cf8d2aa3cca51efe3dddb2ebbe690ba2d2601f2c4a7b6e" +} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 469dcc019..fcfca3bf7 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -19,6 +19,10 @@ on: type: string default: '' required: false + cudaq_dev_images: + type: string + default: '{"amd64-cu12.6":"ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu12.6-gcc12-main","amd64-cu13.0":"ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu13.0-gcc12-main","arm64-cu12.6":"ghcr.io/nvidia/cuda-quantum-devcontainer:arm64-cu12.6-gcc12-main","arm64-cu13.0":"ghcr.io/nvidia/cuda-quantum-devcontainer:arm64-cu13.0-gcc12-main"}' + required: false workflow_dispatch: inputs: deploy_docs: @@ -44,7 +48,7 @@ jobs: build: name: Build runs-on: ${{ startsWith(github.repository, 'NVIDIA/cudaqx') && 'linux-amd64-cpu8' || 'ubuntu-latest' }} - container: ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu12.6-gcc12-main + container: ${{ fromJSON(inputs.cudaq_dev_images || '{"amd64-cu12.6":"ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu12.6-gcc12-main"}')['amd64-cu12.6'] }} permissions: actions: write contents: read diff --git a/.github/workflows/lib_qec.yaml b/.github/workflows/lib_qec.yaml index 40c29284b..fbb00d1a2 100644 --- a/.github/workflows/lib_qec.yaml +++ b/.github/workflows/lib_qec.yaml @@ -15,6 +15,10 @@ on: type: string default: '' required: false + cudaq_dev_images: + type: string + default: '{"amd64-cu12.6":"ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu12.6-gcc12-main","amd64-cu13.0":"ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu13.0-gcc12-main","arm64-cu12.6":"ghcr.io/nvidia/cuda-quantum-devcontainer:arm64-cu12.6-gcc12-main","arm64-cu13.0":"ghcr.io/nvidia/cuda-quantum-devcontainer:arm64-cu13.0-gcc12-main"}' + required: false jobs: build-and-test: @@ -26,7 +30,7 @@ jobs: cuda_version: ['12.6', '13.0'] runs-on: ${{ startsWith(github.repository, 'NVIDIA/cudaqx') && format('linux-{0}-cpu8', matrix.platform) || 'ubuntu-latest' }} container: - image: ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main + image: ${{ fromJSON(inputs.cudaq_dev_images)[format('{0}-cu{1}', matrix.platform, matrix.cuda_version)] }} # SYS_NICE: the default seccomp profile only admits the NUMA memory-policy # syscalls (set_mempolicy/get_mempolicy/mbind) with this capability, which # the hardware-affinity tests exercise. @@ -166,7 +170,7 @@ jobs: cuda_version: ['12.6', '13.0'] runs-on: ${{ startsWith(github.repository, 'NVIDIA/cudaqx') && format('linux-{0}-gpu-a100-latest-1', matrix.platform) || 'ubuntu-latest' }} container: - image: ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main + image: ${{ fromJSON(inputs.cudaq_dev_images)[format('{0}-cu{1}', matrix.platform, matrix.cuda_version)] }} env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} permissions: @@ -261,7 +265,7 @@ jobs: cuda_version: ['13.0'] runs-on: ${{ matrix.runner.label }} container: - image: ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.runner.arch }}-cu${{ matrix.cuda_version }}-gcc12-main + image: ${{ fromJSON(inputs.cudaq_dev_images)[format('{0}-cu{1}', matrix.runner.arch, matrix.cuda_version)] }} # SYS_NICE: see build-and-test. options: --cap-add=SYS_NICE env: diff --git a/.github/workflows/lib_solvers.yaml b/.github/workflows/lib_solvers.yaml index d8ef25b5a..a7c841244 100644 --- a/.github/workflows/lib_solvers.yaml +++ b/.github/workflows/lib_solvers.yaml @@ -15,6 +15,10 @@ on: type: string default: '' required: false + cudaq_dev_images: + type: string + default: '{"amd64-cu12.6":"ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu12.6-gcc12-main","amd64-cu13.0":"ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu13.0-gcc12-main","arm64-cu12.6":"ghcr.io/nvidia/cuda-quantum-devcontainer:arm64-cu12.6-gcc12-main","arm64-cu13.0":"ghcr.io/nvidia/cuda-quantum-devcontainer:arm64-cu13.0-gcc12-main"}' + required: false jobs: build-and-test: @@ -25,7 +29,7 @@ jobs: platform: ['amd64', 'arm64'] cuda_version: ['12.6', '13.0'] runs-on: ${{ startsWith(github.repository, 'NVIDIA/cudaqx') && format('linux-{0}-cpu8', matrix.platform) || 'ubuntu-latest' }} - container: ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main + container: ${{ fromJSON(inputs.cudaq_dev_images)[format('{0}-cu{1}', matrix.platform, matrix.cuda_version)] }} permissions: actions: write contents: read diff --git a/.github/workflows/pr_workflow.yaml b/.github/workflows/pr_workflow.yaml index d7cd39661..e21875c27 100644 --- a/.github/workflows/pr_workflow.yaml +++ b/.github/workflows/pr_workflow.yaml @@ -23,6 +23,7 @@ jobs: build-examples-solvers: ${{ steps.filter.outputs.build-examples-solvers }} pr-number: ${{ steps.export-pr-info.outputs.pr_number }} build-wheels: ${{ steps.filter.outputs.build-wheels }} + cudaq-dev-images: ${{ steps.cudaq-dev-images.outputs.images }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -40,6 +41,11 @@ jobs: run: | echo "pr_number=${{ fromJSON(steps.get-pr-info.outputs.pr-info).number }}" >> $GITHUB_OUTPUT + - name: Set CUDA-Q dev container images + id: cudaq-dev-images + run: | + echo "images=$(jq -c . .github/workflows/cudaq-dev-images.json)" >> "$GITHUB_OUTPUT" + - name: Check what needs testing uses: dorny/paths-filter@v3 id: filter @@ -49,6 +55,7 @@ jobs: build-cudaq: - '.github/actions/get-cudaq-build/**' - '.github/actions/get-cudaq-version/**' + - '.github/workflows/cudaq-dev-images.json' - '.cudaq_version' build-docs: - '.github/workflows/docs.yaml' @@ -122,7 +129,7 @@ jobs: platform: ['amd64', 'arm64'] cuda_version: ['12.6', '13.0'] runs-on: ${{ startsWith(github.repository, 'NVIDIA/cudaqx') && format('linux-{0}-cpu32', matrix.platform) || 'ubuntu-latest' }} - container: ghcr.io/nvidia/cuda-quantum-devcontainer:${{ matrix.platform }}-cu${{ matrix.cuda_version }}-gcc12-main + container: ${{ fromJSON(needs.check-changes.outputs.cudaq-dev-images)[format('{0}-cu{1}', matrix.platform, matrix.cuda_version)] }} permissions: actions: write contents: read @@ -163,6 +170,7 @@ jobs: uses: ./.github/workflows/docs.yaml with: cache_key_suffix: pr${{ needs.check-changes.outputs.pr-number }} + cudaq_dev_images: ${{ needs.check-changes.outputs.cudaq-dev-images }} build-all: name: All libs @@ -173,6 +181,7 @@ jobs: uses: ./.github/workflows/all_libs.yaml with: cache_key_suffix: pr${{ needs.check-changes.outputs.pr-number }} + cudaq_dev_images: ${{ needs.check-changes.outputs.cudaq-dev-images }} build-qec: name: QEC @@ -187,6 +196,7 @@ jobs: uses: ./.github/workflows/lib_qec.yaml with: cache_key_suffix: pr${{ needs.check-changes.outputs.pr-number }} + cudaq_dev_images: ${{ needs.check-changes.outputs.cudaq-dev-images }} build-solvers: name: Solvers @@ -201,6 +211,7 @@ jobs: uses: ./.github/workflows/lib_solvers.yaml with: cache_key_suffix: pr${{ needs.check-changes.outputs.pr-number }} + cudaq_dev_images: ${{ needs.check-changes.outputs.cudaq-dev-images }} build-wheels: name: Wheels diff --git a/cmake/Modules/CUDA-QX.cmake b/cmake/Modules/CUDA-QX.cmake index d4a12b77c..c3011b35e 100644 --- a/cmake/Modules/CUDA-QX.cmake +++ b/cmake/Modules/CUDA-QX.cmake @@ -88,29 +88,15 @@ function(cudaqx_add_device_code LIBRARY_NAME) foreach(source ${ARGS_SOURCES}) get_filename_component(filename ${source} NAME_WE) set(output_file "${CMAKE_CURRENT_BINARY_DIR}/${LIBRARY_NAME}_${filename}.o") + cmake_path(GET output_file FILENAME baseName) - # nvq++ drops intermediates named after the *source* (.o, .qke.o, - # .classic.o, ...) into its working directory, so two targets - # compiling the same source file in the same directory race and corrupt - # each other's objects under parallel builds. Isolate each object's - # compile in its own working directory. - set(work_dir "${CMAKE_CURRENT_BINARY_DIR}/${LIBRARY_NAME}_${filename}.nvqpp") - file(MAKE_DIRECTORY ${work_dir}) - - # TODO: this custom command only depends on the source file, not on the - # headers it includes (nvq++ emits no depfile here), so header changes do - # NOT trigger recompilation -- a stale object silently survives ninja - # after e.g. a config-struct layout change (ABI-mismatch segfaults). - # Until nvq++ depfile output is wired up, `rm` the affected - # _.o under the build tree after header changes. add_custom_command( OUTPUT ${output_file} COMMAND ${COMPILER} ${ARGS_COMPILER_FLAGS} -c -fPIC - ${CMAKE_CURRENT_SOURCE_DIR}/${source} -o ${output_file} + ${CMAKE_CURRENT_SOURCE_DIR}/${source} -o ${baseName} "$<$:-I $>" DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${source} ${ARGS_DEPENDS_ON} - WORKING_DIRECTORY ${work_dir} COMMENT "Compiling ${source} with nvq++" VERBATIM ) @@ -139,7 +125,7 @@ function(_cudaqx_import_nvqir_target target_name library_name) set_target_properties(${target_name} PROPERTIES IMPORTED_LOCATION "${CUDAQ_LIBRARY_DIR}/${library_name}${CMAKE_SHARED_LIBRARY_SUFFIX}" IMPORTED_SONAME "${library_name}${CMAKE_SHARED_LIBRARY_SUFFIX}" - IMPORTED_LINK_INTERFACE_LIBRARIES "cudaq::cudaq-platform-default;cudaq::cudaq-em-default;cudaq::cudaq-mlir-runtime") + IMPORTED_LINK_INTERFACE_LIBRARIES "cudaq::cudaq-platform-default;cudaq::cudaq-em-default") endif() endfunction() diff --git a/docker/build_env/cudaqx.dev.Dockerfile b/docker/build_env/cudaqx.dev.Dockerfile index 7bd15e849..b4f8a200f 100644 --- a/docker/build_env/cudaqx.dev.Dockerfile +++ b/docker/build_env/cudaqx.dev.Dockerfile @@ -6,7 +6,7 @@ # the terms of the Apache License 2.0 which accompanies this distribution. # # ============================================================================ # -ARG base_image=ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu12.6-gcc12-main +ARG base_image=ghcr.io/nvidia/cuda-quantum-devcontainer@sha256:46d0b72f85b083b8fb20af243eeca1308fbdee4fa2e87d1166f029a38d0d72e7 FROM $base_image ARG cuda_version=12.6 @@ -16,9 +16,11 @@ LABEL org.opencontainers.image.source="https://github.com/NVIDIA/cudaqx" LABEL org.opencontainers.image.title="cudaqx-dev" LABEL org.opencontainers.image.url="https://github.com/NVIDIA/cudaqx" +# FIXME: Remove the cmake install once private repos are updated. RUN apt-get update && CUDA_DASH=$(echo $cuda_version | tr '.' '-') \ && apt-get install -y gfortran libblas-dev jq cuda-nvtx-${CUDA_DASH} \ && apt-get install -y git-lfs \ + && python3 -m pip install "cmake<4" --user \ && apt-get autoremove -y --purge && apt-get clean && rm -rf /var/lib/apt/lists/* COPY .cudaq_version /cudaq_version diff --git a/docker/build_env/cudaqx.wheel.Dockerfile b/docker/build_env/cudaqx.wheel.Dockerfile index c07260084..864ccb8db 100644 --- a/docker/build_env/cudaqx.wheel.Dockerfile +++ b/docker/build_env/cudaqx.wheel.Dockerfile @@ -6,7 +6,7 @@ # the terms of the Apache License 2.0 which accompanies this distribution. # # ============================================================================ # -ARG base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-amd64-cu12.6-gcc12-main +ARG base_image=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-amd64-cu12.6-cudaqx-0.15.0 FROM ${base_image} ARG python_version=3.12 diff --git a/docs/sphinx/api/qec/python_realtime_decoding_api.rst b/docs/sphinx/api/qec/python_realtime_decoding_api.rst index 3b4b0e087..4596808ee 100644 --- a/docs/sphinx/api/qec/python_realtime_decoding_api.rst +++ b/docs/sphinx/api/qec/python_realtime_decoding_api.rst @@ -72,82 +72,65 @@ Configuration API The configuration API enables setting up decoders before circuit execution. Decoders are configured using YAML files or programmatically constructed configuration objects. -Decoder Parameters -^^^^^^^^^^^^^^^^^^ - -Decoder-specific parameters (``decoder_config.decoder_custom_args``) are -plain dicts. The set of accepted keys, their types, and which are required -are defined by the *parameter schema* each decoder registers -- including -out-of-tree decoder plugins. Use ``cudaq_qec.decoder_param_schema(name)`` to -inspect a decoder's parameters and ``cudaq_qec.registered_decoder_schemas()`` -to list all decoders with registered schemas. - -For example, the ``pymatching`` decoder accepts ``error_rate_vec`` -(per-error prior probabilities in the range ``(0, 0.5]``, length matching -the decoder ``block_size``) and ``merge_strategy`` (one of ``"disallow"``, -``"independent"``, ``"smallest_weight"``, ``"keep_original"``, -``"replace"``): - -.. code-block:: python - - config.type = "pymatching" - config.decoder_custom_args = { - "error_rate_vec": [0.1, 0.1, 0.1], - "merge_strategy": "smallest_weight", - } - -The ``trt_decoder`` accepts ``onnx_load_path`` or ``engine_load_path`` -(mutually exclusive), ``engine_save_path``, ``precision`` ("fp16", "bf16", -"int8", "fp8", "tf32", "noTF32", or "best"), ``memory_workspace`` (bytes), -``batch_size``, ``use_cuda_graph``, and an optional global decoder attached -via ``global_decoder`` plus ``global_decoder_params`` (a nested dict whose -keys follow the schema of the named global decoder). - -.. py:function:: cudaq_qec.decoder_param_schema(decoder_name) - - Return the registered parameter schema for a decoder as a list of - descriptors (``key``, ``kind``, ``required``, and, for nested sections, - ``subschema`` or ``discriminator``), or ``None`` when the decoder has not - registered one. - -.. py:function:: cudaq_qec.registered_decoder_schemas() - - Names of all decoders (and nested parameter sections) with registered - parameter schemas. - -.. py:function:: cudaq_qec.decoder_config_json_schema() - - Return a JSON Schema (draft 2020-12) document, as a string, that - validates ``multi_decoder_config`` YAML files. Generated from the decoder - parameter schemas registered in this installation (including loaded - third-party decoder plugins), for use with standard tools such as - ``check-jsonschema``, the python ``jsonschema`` package, or editor YAML - language servers. Schema validation hooks are not representable in JSON - Schema, so a passing document may still be rejected when parsed. - -.. py:method:: decoder_config.validate_custom_args() - - Validate ``decoder_custom_args`` against the parameter schema registered - for this decoder ``type``: unknown keys, missing required keys, and the - schema's own validation hook. Raises ``RuntimeError`` on the first - violation. YAML parsing applies the same checks automatically; call this - to vet a configuration built programmatically before using it. Also - available on ``multi_decoder_config`` to validate every decoder at once. - -Deprecated Typed Configuration Classes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The typed configuration classes from earlier releases -(``nv_qldpc_decoder_config``, ``trt_decoder_config``, ``pymatching_config``, -``chromobius_config``, ``multi_error_lut_config``, and the -``qecrt.config``-level ``single_error_lut_config``, ``sliding_window_config``, -and ``srelay_bp_config``) remain available as deprecated compatibility shims. -They emit a ``DeprecationWarning`` on construction and will be removed in a -future release; existing code that builds one and passes it to -``decoder_config.set_decoder_custom_args`` (or assigns it to -``decoder_config.decoder_custom_args``) continues to work unchanged. Note -that *reading* ``decoder_custom_args`` now always returns a plain dict, never -a typed object. New code should assign dicts directly, as shown above. +Configuration Types +^^^^^^^^^^^^^^^^^^^ + +.. py:class:: cudaq_qec.pymatching_config + + Configuration for the PyMatching decoder in the real-time decoding system. + Use this with ``decoder_config.type = "pymatching"``. + Set ``decoder_config.type`` before passing this object to + ``decoder_config.set_decoder_custom_args``. + The decoder input matrix must be graphlike: each ``H_sparse`` column can + contain only one or two detector entries. + + **Attributes:** + + .. py:attribute:: error_rate_vec + :type: Optional[List[float]] + + Per-error prior probabilities. When provided, the length must match the + decoder ``block_size`` and each value must be in the range ``(0, 0.5]``. + + .. py:attribute:: merge_strategy + :type: Optional[str] + + PyMatching edge merge strategy. Supported values are ``"disallow"``, + ``"independent"``, ``"smallest_weight"``, ``"keep_original"``, and + ``"replace"``. + +.. py:class:: cudaq_qec.trt_decoder_config + + Configuration for TensorRT decoder in real-time decoding system. + + **Attributes:** + + .. py:attribute:: onnx_load_path + :type: Optional[str] + + Path to ONNX model file. Mutually exclusive with engine_load_path. + + .. py:attribute:: engine_load_path + :type: Optional[str] + + Path to pre-built TensorRT engine file. Mutually exclusive with + onnx_load_path. + + .. py:attribute:: engine_save_path + :type: Optional[str] + + Path to save built TensorRT engine for reuse. + + .. py:attribute:: precision + :type: Optional[str] + + Inference precision mode: "fp16", "bf16", "int8", "fp8", "tf32", + "noTF32", or "best" (default). + + .. py:attribute:: memory_workspace + :type: Optional[int] + + Workspace memory size in bytes (default: 1073741824 = 1GB). Configuration Functions ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/sphinx/examples/qec/cpp/real_time_complete.cpp b/docs/sphinx/examples/qec/cpp/real_time_complete.cpp index ffcd77c8c..373c9c81f 100644 --- a/docs/sphinx/examples/qec/cpp/real_time_complete.cpp +++ b/docs/sphinx/examples/qec/cpp/real_time_complete.cpp @@ -45,14 +45,9 @@ void save_dem(const cudaq::qec::detector_error_model &dem, config.D_sparse = cudaq::qec::generate_timelike_sparse_detector_matrix( numSyndromesPerRound, numRounds, false); - // Decoder parameters are a plain heterogeneous_map; keys are governed by - // the parameter schema the decoder registered. - cudaqx::heterogeneous_map lut_args; - lut_args.insert("lut_error_depth", 2); - config.decoder_custom_args = lut_args; - // Check the map against the decoder's schema (unknown keys, missing - // required keys, decoder-specific constraints) before using the config. - config.validate_custom_args(); + cudaq::qec::decoding::config::multi_error_lut_config lut_config; + lut_config.lut_error_depth = 2; + config.decoder_custom_args = lut_config; cudaq::qec::decoding::config::multi_decoder_config multi_config; multi_config.decoders.push_back(config); diff --git a/docs/sphinx/examples/qec/python/real_time_complete.py b/docs/sphinx/examples/qec/python/real_time_complete.py index 4ce88c137..5309065a1 100644 --- a/docs/sphinx/examples/qec/python/real_time_complete.py +++ b/docs/sphinx/examples/qec/python/real_time_complete.py @@ -107,12 +107,9 @@ def main(): 0] // num_syndromes_per_round + 1 config.D_sparse = qec.generate_timelike_sparse_detector_matrix( num_syndromes_per_round, num_rounds, False) - # Decoder parameters are a plain dict; keys are governed by the parameter - # schema the decoder registered (see qec.decoder_param_schema). - config.decoder_custom_args = {"lut_error_depth": 2} - # Check the dict against the decoder's schema (unknown keys, missing - # required keys, decoder-specific constraints) before using the config. - config.validate_custom_args() + lut_config = qec.multi_error_lut_config() + lut_config.lut_error_depth = 2 + config.set_decoder_custom_args(lut_config) multi_config = qec.multi_decoder_config() multi_config.decoders = [config] diff --git a/docs/sphinx/examples_rst/qec/realtime_decoding.rst b/docs/sphinx/examples_rst/qec/realtime_decoding.rst index 00bccc5cd..2f5cff664 100644 --- a/docs/sphinx/examples_rst/qec/realtime_decoding.rst +++ b/docs/sphinx/examples_rst/qec/realtime_decoding.rst @@ -121,7 +121,7 @@ base decoder to accumulate logical corrections returned by ``get_corrections``. Vanilla PyMatching requires graphlike detector error models, where every ``H_sparse`` column has one or two detector entries. For belief propagation decoders, the user sets iteration limits and convergence criteria. -Decoder parameters are validated against the parameter schema each decoder registers, ensuring unknown keys are rejected and required parameters are present. +The configuration API provides type-safe structures for each decoder, ensuring that all required parameters are included. The configuration is then saved to a YAML file for reuse. The YAML format is human-readable, making it easy to inspect, modify, and share configurations across different execution environments. @@ -139,10 +139,10 @@ For example, a PyMatching real-time decoder can be configured programmatically: config.D_sparse = qec.generate_timelike_sparse_detector_matrix( num_syndromes_per_round, num_rounds, include_first_round=False) - config.decoder_custom_args = { - "error_rate_vec": list(dem.error_rates), - "merge_strategy": "smallest_weight", - } + pm_config = qec.pymatching_config() + pm_config.error_rate_vec = list(dem.error_rates) + pm_config.merge_strategy = "smallest_weight" + config.set_decoder_custom_args(pm_config) multi_config = qec.multi_decoder_config() multi_config.decoders = [config] @@ -165,74 +165,6 @@ arguments: error_rate_vec: [ 0.1, 0.1, 0.1 ] merge_strategy: smallest_weight -The ``decoder_custom_args`` section is converted between YAML and the -parameter map a decoder's constructor receives using a *parameter schema* -registered under the decoder's name. All built-in decoders ship with a -schema, and custom (out-of-tree) decoder plugins can register their own so -their parameters become configurable through the same YAML -- no changes to -the CUDA-Q QEC libraries are required. A plugin registers its schema from a -static initializer in the same shared library that registers the decoder -itself (see ``cudaq/qec/decoder_config_schema.h`` and the in-tree example -plugin ``single_error_lut_example``): - -.. code-block:: cpp - - #include "cudaq/qec/decoder_config_schema.h" - - namespace { - struct schema_registrar { - schema_registrar() { - using k = cudaq::qec::decoding::config::param_kind; - cudaq::qec::decoding::config::decoder_schema schema{ - "my_decoder", - { - {"strength", k::f64}, - {"passes", k::int32}, - {"mode", k::string, /*required=*/true}, - }}; - // Optional: cross-field constraints the per-key specs can't express. - // Unknown keys and missing required keys are already rejected by the - // framework; a decoder never implements those checks itself. - schema.validate = [](const cudaqx::heterogeneous_map &args) { - if (args.contains("strength") && args.get("strength") <= 0.0) - throw std::runtime_error("my_decoder: strength must be positive"); - }; - cudaq::qec::decoding::config::register_decoder_schema( - std::move(schema)); - } - }; - schema_registrar register_schema; - } // namespace - -With the schema in place, a ``decoder_custom_args`` section for -``type: my_decoder`` is validated (unknown keys and missing required keys are -rejected, then the schema's ``validate`` hook runs) and delivered to the -decoder's constructor as a ``cudaqx::heterogeneous_map``. The same checks can -be applied to a configuration built programmatically -- before it is -serialized or used -- by calling ``decoder_config::validate_custom_args()`` -(``config.validate_custom_args()`` in Python, also available on -``multi_decoder_config``). The registered schemas can be inspected from -Python via ``qec.decoder_param_schema("my_decoder")`` and -``qec.registered_decoder_schemas()``. - -The registered schemas can also be exported as a standard JSON Schema -(draft 2020-12) document via ``qec.decoder_config_json_schema()``, so -configuration YAML files can be validated by third-party tooling -- editors, -CI checks, or the `check-jsonschema -`_ command line tool -- without -loading the CUDA-Q QEC libraries: - -.. code-block:: bash - - python3 -c "import cudaq_qec; print(cudaq_qec.decoder_config_json_schema())" > decoder_config_schema.json - check-jsonschema --schemafile decoder_config_schema.json my_config.yaml - -The export is generated from the schemas registered at call time, so decoder -plugins loaded in the process (including out-of-tree ones) appear in it -automatically. Schema ``validate`` hooks are arbitrary code and cannot be -represented in JSON Schema, so a file that passes the exported schema may -still be rejected by a hook when the configuration is parsed. - ``cuda_device_id`` pins a GPU-accelerated decoder (e.g. ``nv-qldpc-decoder`` or ``trt_decoder``) to a specific CUDA device. The same knob is available as a construction parameter in C++ and Python @@ -657,8 +589,8 @@ Decoder Selection ^^^^^^^^^^^^^^^^^ The page `CUDA-Q QEC Decoders `_ provides information about which decoders are compatible with real-time decoding. -The TRT decoder (``trt_decoder``) can be configured for real-time decoding by specifying -its ``decoder_custom_args`` parameters. This is useful for neural network-based +The TRT decoder (``trt_decoder``) can be configured for real-time decoding by specifying +``trt_decoder_config`` parameters. This is useful for neural network-based decoders trained for specific codes and noise models. Note that TRT models must be trained with the appropriate input/output dimensions matching the syndrome and error spaces. See :ref:`trt_decoder_api_python` for detailed configuration options. diff --git a/docs/sphinx/examples_rst/qec/realtime_relay_bp.rst b/docs/sphinx/examples_rst/qec/realtime_relay_bp.rst index f4a4e0a75..aabdd4e8d 100644 --- a/docs/sphinx/examples_rst/qec/realtime_relay_bp.rst +++ b/docs/sphinx/examples_rst/qec/realtime_relay_bp.rst @@ -7,44 +7,14 @@ Relay BP Decoding with CUDA-Q Realtime from source and is not part of any distributed CUDA-Q QEC binaries. This guide explains how to build, test, and run the nv-qldpc-decoder Relay BP -decoder using CUDA-Q's realtime dispatch system. The decoder is driven by a -**self-relaunching device-graph scheduler** and can operate in three +decoder using CUDA-Q's realtime host dispatch system. The decoder runs as a +CPU-launched CUDA graph (``HOST_LOOP`` dispatch path) and can operate in three configurations: - **CI unit test** -- standalone executable, no FPGA or network hardware needed - **Emulated end-to-end test** -- software FPGA emulator replaces real hardware - **FPGA end-to-end test** -- real FPGA connected via ConnectX RDMA/RoCE -Decode dispatch architecture -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The realtime path uses the per-round decode-server protocol with three RPCs: -``enqueue_syndromes`` (append one round of syndromes), ``get_corrections`` -(read the logical correction for a completed shot), and ``reset_decoder``. -These are serviced by a single GPU **device-graph scheduler** -- a persistent, -self-relaunching CUDA graph: - -- All three RPCs are ``DEVICE_CALL`` handlers. ``enqueue_syndromes`` - accumulates a round's syndromes into the decoder's device-resident state; - when a full window has accumulated it returns a sentinel - (``CUDAQ_DISPATCH_STATUS_TRIGGER_GRAPH``) that tells the scheduler to fire - the decode. -- The Relay BP decode is captured as a **device-launchable cooperative CUDA - graph** and launched *fire-and-forget* from the scheduler when a window is - ready. ``get_corrections`` then reads the result. -- After firing a decode the scheduler **tail self-relaunches** - (``cudaGraphLaunch(cudaGetCurrentGraphExec(), cudaStreamGraphTailLaunch)``), - which resets the 120 fire-and-forget-launch-per-parent-execution budget -- - so an unbounded number of decodes can be dispatched without the host in the - loop. The tail launch is ordered after the fired decode, so - ``get_corrections`` always observes the finished result. - -This replaces the earlier ``HOST_LOOP`` design (a CPU thread launching one -graph per request). ``libcudaq-realtime`` provides the scheduler -(``cudaq_create_dispatch_graph_regular`` / ``cudaq_launch_dispatch_graph`` in -``dispatch_kernel.cu``); the closed-source proprietary archive provides the -``DEVICE_CALL`` handlers (see *Obtaining the proprietary components* below). - Prerequisites ------------- @@ -84,26 +54,11 @@ Software three configurations -- see *Obtaining the nv-qldpc-decoder plugin* below for how to install it. -Obtaining the proprietary components +Obtaining the nv-qldpc-decoder plugin ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The realtime decode path uses **two** closed-source artifacts that are not -built from this (cudaqx) repository: - -- ``libcudaq-qec-nv-qldpc-decoder.so`` -- the Relay BP decoder **plugin**, - ``dlopen``'d at runtime. It supplies the device-launchable cooperative - decode graph (``capture_decode_graph``). -- ``libcudaq-qec-realtime-cudevice-proprietary.a`` -- a static **archive** - needed at **build** time. It contains the ``enqueue_syndromes`` / - ``get_corrections`` / ``reset_decoder`` ``DEVICE_CALL`` handlers (the device - functions the scheduler dispatches). It is linked ``WHOLE_ARCHIVE`` and - device-linked into the bridge and the CI test, and is pointed at via the - ``-DCUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE=`` CMake variable. - Both artifacts come from the same closed-source decoder package; build the - ``cudaq-qec-realtime-cudevice-proprietary`` target from the proprietary - decoder sources to produce the ``.a``. - -The plugin must be obtained as a pre-built binary as shown below. +The ``libcudaq-qec-nv-qldpc-decoder.so`` plugin is closed-source and is not +built from this repository. It must be obtained as a pre-built binary. .. important:: @@ -157,22 +112,14 @@ Source Repositories - ``main`` branch (or your feature branch) * - **cuda-quantum** (realtime) - https://github.com/NVIDIA/cuda-quantum - - Branch ``releases/v0.15.1`` + - Branch ``releases/v0.14.1`` * - **holoscan-sensor-bridge** - https://github.com/nvidia-holoscan/holoscan-sensor-bridge - Tag ``2.6.0-EA2`` -``cuda-quantum`` provides ``libcudaq-realtime`` (the dispatch kernel, ring -buffer management, and the device-graph scheduler). ``holoscan-sensor-bridge`` -provides the Hololink ``GpuRoceTransceiver`` library for RDMA transport. - -.. note:: - - The self-relaunching device-graph scheduler is provided by the - ``releases/v0.15.1`` branch of ``cuda-quantum`` (the extension that adds the - ``CUDAQ_DISPATCH_STATUS_TRIGGER_GRAPH`` sentinel, the triggered - fire-and-forget decode launch, and tail self-relaunch on top of the - device-side graph dispatch). +``cuda-quantum`` provides ``libcudaq-realtime`` (the host dispatcher, ring +buffer management, and dispatch kernel). ``holoscan-sensor-bridge`` provides +the Hololink ``GpuRoceTransceiver`` library for RDMA transport. .. note:: @@ -222,21 +169,17 @@ If you only need to run the CI unit test, you can build without # 1. Build libcudaq-realtime git clone https://github.com/NVIDIA/cuda-quantum.git cudaq-realtime-src cd cudaq-realtime-src - git checkout releases/v0.15.1 + git checkout releases/v0.14.1 cd realtime && mkdir -p build && cd build cmake -G Ninja -DCMAKE_INSTALL_PREFIX=/tmp/cudaq-realtime .. ninja && ninja install cd ../../.. - # 2. Build cudaqx with the nv-qldpc-decoder test. - # CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE points at the static - # archive with the DEVICE_CALL handlers; it is linked WHOLE_ARCHIVE into - # the test (see "Obtaining the proprietary components"). + # 2. Build cudaqx with the nv-qldpc-decoder test cmake -S cudaqx -B cudaqx/build \ -DCMAKE_BUILD_TYPE=Release \ -DCUDAQ_DIR=/path/to/cudaq-install/lib/cmake/cudaq/ \ -DCUDAQ_REALTIME_ROOT=/tmp/cudaq-realtime \ - -DCUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE=/path/to/libcudaq-qec-realtime-cudevice-proprietary.a \ -DCUDAQX_ENABLE_LIBS="qec" \ -DCUDAQX_INCLUDE_TESTS=ON cmake --build cudaqx/build --target test_realtime_qldpc_graph_decoding @@ -254,7 +197,7 @@ To also build the bridge and playback tools for emulated or FPGA testing: cd cudaq-realtime-src git sparse-checkout init --cone git sparse-checkout set realtime - git checkout releases/v0.15.1 + git checkout releases/v0.14.1 cd .. # 2. Build holoscan-sensor-bridge (tag 2.6.0-EA2) @@ -297,14 +240,11 @@ To also build the bridge and playback tools for emulated or FPGA testing: ninja && ninja install cd ../../.. - # 4. Build cudaqx with Hololink tools enabled. - # CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE supplies the DEVICE_CALL - # handlers (WHOLE_ARCHIVE-linked into the bridge + test). + # 4. Build cudaqx with Hololink tools enabled cmake -S cudaqx -B cudaqx/build \ -DCMAKE_BUILD_TYPE=Release \ -DCUDAQ_DIR=/path/to/cudaq-install/lib/cmake/cudaq/ \ -DCUDAQ_REALTIME_ROOT=/tmp/cudaq-realtime \ - -DCUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE=/path/to/libcudaq-qec-realtime-cudevice-proprietary.a \ -DCUDAQX_ENABLE_LIBS="qec" \ -DCUDAQX_INCLUDE_TESTS=ON \ -DCUDAQX_QEC_ENABLE_HOLOLINK_TOOLS=ON \ @@ -332,19 +272,14 @@ CI Unit Test ------------ The CI unit test (``test_realtime_qldpc_graph_decoding``) exercises the full -device-graph scheduler decode path without any network hardware. It: +host dispatch decode path without any network hardware. It: 1. Loads the Relay BP config and syndrome data from YAML/text files 2. Creates the decoder via the ``decoder::get("nv-qldpc-decoder", ...)`` plugin API -3. Constructs a ``qec_realtime_session``, which captures the decoder's - device-launchable cooperative decode graph and starts the device-graph - scheduler on a pinned-mapped ring (3 ``DEVICE_CALL`` entries: - ``enqueue_syndromes`` / ``get_corrections`` / ``reset_decoder``) -4. Drives the per-round protocol with ``rpc_producer``: for each shot it sends - one ``enqueue_syndromes`` per round, then a ``get_corrections``; the - scheduler fires the decode when a window completes and tail self-relaunches -5. Verifies each shot's correction against the fixture, then a final - ``reset_decoder`` + ``get_corrections`` confirms reset +3. Captures a CUDA graph of the decode pipeline +4. Wires ``libcudaq-realtime``'s host dispatcher (HOST_LOOP) to a ring buffer +5. Writes RPC requests into the ring buffer, the host dispatcher launches the + CUDA graph, and the test verifies corrections Running ^^^^^^^ @@ -354,7 +289,7 @@ Running cd cudaqx/build # The nv-qldpc-decoder plugin must be in /lib/decoder-plugins/ - # before running -- see "Obtaining the proprietary components" above. + # before running -- see "Obtaining the nv-qldpc-decoder plugin" above. ./libs/qec/unittests/test_realtime_qldpc_graph_decoding @@ -363,56 +298,13 @@ Expected output: .. code-block:: text [==========] Running 1 test from 1 test suite. - [----------] 1 test from GraphDecodeTest - [ RUN ] GraphDecodeTest.DecodesAllSyndromes + [----------] 1 test from RealtimeQLDPCGraphDecodingTest + [ RUN ] RealtimeQLDPCGraphDecodingTest.DispatchHostLoopAllShots ... - [ OK ] GraphDecodeTest.DecodesAllSyndromes (XXX ms) + [ OK ] RealtimeQLDPCGraphDecodingTest.DispatchHostLoopAllShots (XXX ms) [==========] 1 test from 1 test suite ran. [ PASSED ] 1 test. -Surface Code Test (Relay BP) ----------------------------- - -The ``surface_code-1-local`` app example drives the device-graph scheduler -through the in-process RPC path (``CUDAQ_QEC_REALTIME_MODE=inproc_rpc``) with -the nv-qldpc-decoder configured for Relay BP (``--use-relay-bp``). It simulates -a surface code with ``stim`` and generates syndromes on the fly, so -- unlike -the fixed-fixture CI unit test -- it can run an arbitrary number of shots. - -Build the app example (it links the same plugin + proprietary archive as the -CI test): - -.. code-block:: bash - - cmake --build cudaqx/build --target surface_code-1-local - -Run it in two steps -- generate the decoder config (DEM), then run the decode -loop through the scheduler: - -.. code-block:: bash - - cd cudaqx/build - export CUDAQ_DEFAULT_SIMULATOR=stim - export CUDAQ_QEC_REALTIME_MODE=inproc_rpc - - APP=./libs/qec/unittests/realtime/app_examples/surface_code-1-local - - # 1. Generate the Relay BP decoder config (DEM) for a distance-3 surface code - "$APP" --distance 3 --num_rounds 12 --decoder_window 6 \ - --decoder_type nv-qldpc-decoder --use-relay-bp \ - --num_shots 1000 --save_dem config.yml - - # 2. Run the decode loop through the device-graph scheduler - "$APP" --distance 3 --num_rounds 12 --decoder_window 6 \ - --decoder_type nv-qldpc-decoder --use-relay-bp \ - --num_shots 1000 --load_dem config.yml - -A clean run exits ``0`` and reports a small number of non-zero syndrome -measurements alongside a larger number of corrections found. The -``app_examples`` CTest ``surface_code-1-local-test-distance-3-inproc-rpc`` -wraps this flow (it sets ``CUDAQ_QEC_REALTIME_MODE=inproc_rpc`` and -``EXTRA_CLI_ARGS=--use-relay-bp``). - Emulated End-to-End Test ------------------------ @@ -421,20 +313,10 @@ processes run concurrently: 1. **Emulator** -- receives syndromes via the UDP control plane, sends them to the bridge via RDMA, and captures corrections -2. **Bridge** -- runs the device-graph scheduler on the GPU directly on the - Hololink DOCA ring (the scheduler polls the RX flags written by the - Hololink RX kernel and writes responses for the TX kernel), firing the - cooperative Relay BP decode fire-and-forget per completed shot +2. **Bridge** -- runs the host dispatcher and CUDA graph decode loop on the GPU, + receiving syndromes and sending corrections via RDMA 3. **Playback** -- loads syndrome data into the emulator's BRAM and triggers - playback in **per-round** mode (``--per-round``: N ``enqueue_syndromes`` - frames + one ``get_corrections`` per shot), then verifies corrections - -.. note:: - - The orchestration script drives the playback tool in ``--per-round`` mode - automatically (matching the decode-server protocol the scheduler speaks). - The playback tool also retains a shot-based default for other decoders; the - per-round path is opt-in via ``--per-round``. + playback, then verifies corrections Requirements ^^^^^^^^^^^^ @@ -499,16 +381,8 @@ Running --fpga-ip 192.168.0.2 \ --gpu 2 \ --page-size 512 \ - --spacing 100 \ --hsb-dir /path/to/holoscan-sensor-bridge -``--spacing`` is **important for the FPGA** (it is not needed for the -emulator). The FPGA's BRAM player is **open-loop** -- it transmits a frame -every ``--spacing`` microseconds on a fixed hardware timer, with no -backpressure -- whereas the emulator naturally paces itself by waiting for each -response. Without adequate spacing the FPGA outruns the decoder, the input -ring fills, and frames are lost. See the note below for sizing. - Key parameters for FPGA mode: .. list-table:: @@ -528,26 +402,15 @@ Key parameters for FPGA mode: * - ``--page-size`` - Ring buffer slot size in bytes (use ``512`` on GB200 for alignment) * - ``--spacing`` - - Inter-**frame** spacing in microseconds (FPGA BRAM-player timer) + - Inter-shot spacing in microseconds .. note:: - **Sizing the spacing.** In per-round mode each shot is ``rounds + 1`` - frames (N ``enqueue_syndromes`` + one ``get_corrections``) but only one - decode, so the decoder consumes roughly one ``decode_time`` per shot. Since - ``--spacing`` is the gap between *frames*, the sustained-safe value is - - .. code-block:: text - - spacing >= decode_time / (rounds + 1) - - For this ``[[8,3,6]]`` relay-BP config (~200 us decode, 4 rounds -> 5 - frames/shot) that is ``>= ~40 us``. Start **conservative** (e.g. - ``--spacing 100``) for the first run to rule out ring overrun while - confirming corrections, then tune down toward ``~50 us`` for a realistic - latency profile. If frames are still dropped/duplicated at generous - spacing, the cause is *not* ring overrun -- investigate the FPGA capture - (ILA) side. + The ``--spacing`` value should be set to at least the per-shot decode + time to avoid overrunning the input ring buffer. If syndromes arrive faster + than the decoder can process them, the buffer fills up and messages are lost. + Use a ``--spacing`` value at or above the observed decode time for sustained + operation. GPU Selection ^^^^^^^^^^^^^ @@ -680,6 +543,9 @@ Run Options * - ``--page-size N`` - ``384`` - Ring buffer slot size in bytes + * - ``--num-pages N`` + - ``128`` + - Number of ring buffer slots * - ``--spacing N`` - ``10`` - Inter-shot spacing in microseconds @@ -689,24 +555,3 @@ Run Options * - ``--control-port N`` - ``8193`` - UDP control port for emulator - -Ring buffer depth (``num_pages``) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The ring depth is intentionally **not** a script option and is fixed at -**64** (both the bridge and playback default to it). This matches the -Hololink ``gpu_roce_transceiver`` work-queue depth ``WQE_NUM = 64``: the -transceiver posts 64 receive/send WQEs and runs one kernel thread per WQE. - -A ring deeper than ``WQE_NUM`` makes a single transceiver thread service more -than one ring slot (slot ``t`` and slot ``t+64`` share one WQE / CQ position), -and the free-running RX/TX kernels then race on that shared resource. On the -emulator this was observed as a rare (~1-2%) **duplicated frame ``W`` plus a -dropped frame ``W+64``** -- every failure was an exact ``(W, W+64)`` pair on a -single thread. A 1:1 slot-to-WQE mapping (``num_pages <= WQE_NUM``) is the -only safe configuration and is collision-free. - -The bridge enforces this: if ``--num-pages`` is ever passed with a value above -``WQE_NUM``, it clamps to 64 and prints a warning. Supporting a deeper ring -would require changing ``WQE_NUM`` (and the per-thread WQE striding) in -``holoscan-sensor-bridge``, diverging from the ``2.6.0-EA2`` tag. diff --git a/libs/qec/CMakeLists.txt b/libs/qec/CMakeLists.txt index c452eff0a..1740fb807 100644 --- a/libs/qec/CMakeLists.txt +++ b/libs/qec/CMakeLists.txt @@ -287,7 +287,6 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/lib/version.cpp.in" # ============================================================================== add_subdirectory(lib) -add_subdirectory(tools) if (CUDAQX_QEC_BINDINGS_PYTHON) add_subdirectory(python) @@ -344,12 +343,6 @@ function(add_target_libs_to_wheel decoder_lib) install(FILES ${decoder_lib} DESTINATION ${CMAKE_INSTALL_LIBDIR}/decoder-plugins COMPONENT qec-lib-plugins) - # Mirror the (rpath-patched) plugin into the build tree so build-tree ctests - # can load it -- the runtime scans /decoder-plugins next to - # libcudaq-qec.so. configure_file re-runs cmake when the source .so changes. - get_filename_component(_decoder_lib_name "${decoder_lib}" NAME) - configure_file("${decoder_lib}" - "${CMAKE_BINARY_DIR}/lib/decoder-plugins/${_decoder_lib_name}" COPYONLY) endfunction() if(NOT QEC_EXTERNAL_DECODERS) diff --git a/libs/qec/include/cudaq/qec/decoder_config_schema.h b/libs/qec/include/cudaq/qec/decoder_config_schema.h deleted file mode 100644 index a6a9f378f..000000000 --- a/libs/qec/include/cudaq/qec/decoder_config_schema.h +++ /dev/null @@ -1,153 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2024 - 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include "cuda-qx/core/heterogeneous_map.h" -#include -#include -#include - -namespace cudaq::qec::decoding::config { - -/// Decoder parameter schemas. -/// -/// A decoder that wants its constructor parameters to be configurable through -/// the realtime decoding YAML (the `decoder_custom_args` section of a -/// `decoder_config`) registers a `decoder_schema` describing those parameters, -/// keyed by the same name it registers its `cudaq::qec::decoder` extension -/// under. The YAML layer uses the schema to convert the section to and from a -/// `cudaqx::heterogeneous_map` generically -- no decoder-specific parsing code -/// is required in the framework, so out-of-tree decoder plugins can make -/// themselves "realtime ready" from their own shared library. -/// -/// The value kinds are a deliberately small, closed set of canonical storage -/// types. Values parsed from YAML are stored in the map with exactly the type -/// listed for their kind; `heterogeneous_map::get` handles related-type -/// retrieval (e.g. an `int` stored for an `int32` parameter can be read back -/// as `std::size_t`). -enum class param_kind { - boolean, ///< stored as bool - int32, ///< stored as int - uint64, ///< stored as std::size_t - f64, ///< stored as double - string, ///< stored as std::string - f64_vec, ///< stored as std::vector - f64_matrix, ///< stored as std::vector> - /// Nested mapping parsed with the fixed schema named by - /// `param_spec::subschema`; stored as a nested heterogeneous_map. - subschema, - /// Nested mapping whose schema is selected at parse time by the value of - /// the sibling string key named by `param_spec::discriminator` (e.g. - /// `global_decoder_params` is parsed with the schema registered under the - /// value of `global_decoder`); stored as a nested heterogeneous_map. - discriminated, -}; - -/// One parameter of a decoder's custom-args schema. -struct param_spec { - std::string key; - param_kind kind = param_kind::string; - /// Parsing fails when a required key is absent. - bool required = false; - /// For kind::subschema: the registered schema name to parse the nested - /// mapping with. - std::string subschema; - /// For kind::discriminated: the sibling key whose (string) value names the - /// schema to parse the nested mapping with. - std::string discriminator; - /// For kind::discriminated: when the discriminator is present and names a - /// registered schema but this key is absent, insert an empty nested map so - /// downstream consumers see the section (matches the trt_decoder - /// global_decoder_params defaulting behavior). - bool materialize_empty = false; -}; - -/// Declarative description of a decoder's `decoder_custom_args` section. -struct decoder_schema { - /// Registry key. For decoders this must match the name the decoder is - /// registered under (the YAML `type` value). Schemas that only serve as - /// nested sections (e.g. "srelay_bp") may use any unique name. - std::string name; - std::vector params; - /// Optional cross-field validation hook, invoked after a section has been - /// parsed and its required keys checked. Throw std::runtime_error to - /// reject the configuration. - std::function validate; -}; - -/// Register (or replace) a schema. Decoder plugins call this from a static -/// initializer in the same shared library that registers the decoder itself; -/// the plugin loader runs before any configuration is parsed. -/// -/// Schemas are never unregistered or unloaded, by design: registered schemas -/// (including their validate hooks) stay in the registry for the remainder of -/// the process, even after the registering plugin is dlclose'd at exit. The -/// registry is intentionally leaked so no schema destructor can run after its -/// plugin's code has been unloaded. -__attribute__((visibility("default"))) void -register_decoder_schema(decoder_schema schema); - -/// Look up a schema by name. Returns nullptr when no schema is registered -/// under `name`. The returned pointer remains valid for the process lifetime -/// (schemas are never unregistered; see register_decoder_schema). -__attribute__((visibility("default"))) const decoder_schema * -find_decoder_schema(const std::string &name); - -/// Names of all registered schemas (for diagnostics and introspection). -__attribute__((visibility("default"))) std::vector -registered_decoder_schema_names(); - -/// Validate a custom-args map against the schema registered under -/// `schema_name`: every key must appear in the schema (nested sections are -/// checked recursively), every required key must be present, and the schema's -/// `validate` hook (if any) runs last. Throws std::runtime_error describing -/// the first violation. Maps that did not come from the YAML parser (e.g. -/// built programmatically or from a Python dict) get the same checks the -/// parser applies, so configurations can be validated before use. A map for -/// a name with no registered schema is rejected unless it is empty. -__attribute__((visibility("default"))) void -validate_custom_args(const std::string &schema_name, - const cudaqx::heterogeneous_map &args); - -/// Overload taking a resolved schema and an error-message context (e.g. -/// "decoder_custom_args (trt_decoder)"). This is the single canonical walk; -/// the YAML parser applies it after parsing a section. -__attribute__((visibility("default"))) void -validate_custom_args(const decoder_schema &schema, - const cudaqx::heterogeneous_map &args, - const std::string &context); - -/// Apply schema-declared defaults to `args` (recursively): a discriminated -/// section with `materialize_empty` set is inserted as an empty nested map -/// when its discriminator names a registered schema and the section key is -/// absent. Idempotent. The YAML parser applies this to parsed sections, and -/// decoder_config::decoder_custom_args_to_heterogeneous_map() applies it to -/// programmatically built configs so both paths hand decoders the same map. -__attribute__((visibility("default"))) void -materialize_default_args(const decoder_schema &schema, - cudaqx::heterogeneous_map &args); - -/// Remove (with a warning per key) every key `args` holds that is not in -/// `schema`, recursing into nested sections whose schemas resolve. Applied by -/// decoder_config::decoder_custom_args_to_heterogeneous_map() so the map a -/// local decoder's constructor receives and the map serialized to YAML for a -/// remote target are the same map -- a non-schema key can never take effect -/// locally but silently vanish remotely. -__attribute__((visibility("default"))) void -drop_non_schema_keys(const decoder_schema &schema, - cudaqx::heterogeneous_map &args); - -/// Deep equality over the canonical value kinds stored in custom-args maps -/// (scalars, double vectors/matrices, and nested maps). Values of other -/// types compare unequal. -__attribute__((visibility("default"))) bool -custom_args_maps_equal(const cudaqx::heterogeneous_map &a, - const cudaqx::heterogeneous_map &b); - -} // namespace cudaq::qec::decoding::config diff --git a/libs/qec/include/cudaq/qec/realtime/decoder_rpc_ids.h b/libs/qec/include/cudaq/qec/realtime/decoder_rpc_ids.h index 67fcc2cb4..347a9bd3d 100644 --- a/libs/qec/include/cudaq/qec/realtime/decoder_rpc_ids.h +++ b/libs/qec/include/cudaq/qec/realtime/decoder_rpc_ids.h @@ -15,9 +15,6 @@ namespace cudaq::qec::decoding::rpc { -// The QEC multi-handler accumulator pattern is fixed by -// proposals/decoder_server_runtime.md. All producers and consumers must use -// these canonical function IDs and payload shapes. constexpr std::uint32_t kEnqueueSyndromesFunctionId = cudaq::realtime::fnv1a_hash("enqueue_syndromes"); constexpr std::uint32_t kGetCorrectionsFunctionId = @@ -40,7 +37,7 @@ struct __attribute__((packed)) EnqueueRequestPayload { std::int64_t counter; ///< arg1 std::int64_t syndrome_mapping_id; ///< arg2 std::int64_t num_syndromes; ///< arg3 (# syndrome bits following) - // Trailing: ceil(num_syndromes/8) bit-packed bytes (LSB-first), no pad. + // Trailing: ceil(num_syndromes/8) bit-packed bytes + 0..7 zero pad. }; static_assert(sizeof(EnqueueRequestPayload) == 32, "EnqueueRequestPayload must be exactly 32 bytes per " @@ -48,14 +45,12 @@ static_assert(sizeof(EnqueueRequestPayload) == 32, struct __attribute__((packed)) GetCorrectionsRequestPayload { std::int64_t decoder_id; ///< arg0 - std::int64_t return_size; ///< arg1 (# correction bits to fetch; the - ///< cc.device_call lowering serializes the - ///< OUT std::vector length here) - std::uint8_t reset; ///< arg2 (0 = keep state, 1 = reset after read; - ///< trailing bool, no padding) + std::int64_t return_size; ///< arg1 (# correction bits to fetch) + std::uint8_t reset; ///< arg2 (0 = keep state, 1 = reset after read) + std::uint8_t _pad[7]; ///< zero pad to 8-byte multiple }; -static_assert(sizeof(GetCorrectionsRequestPayload) == 17, - "GetCorrectionsRequestPayload must be exactly 17 bytes per " +static_assert(sizeof(GetCorrectionsRequestPayload) == 24, + "GetCorrectionsRequestPayload must be exactly 24 bytes per " "decoder_server_runtime.md#get_corrections"); struct __attribute__((packed)) ResetRequestPayload { @@ -65,26 +60,12 @@ static_assert(sizeof(ResetRequestPayload) == 8, "ResetRequestPayload must be exactly 8 bytes per " "decoder_server_runtime.md#reset_decoder"); -#ifdef __CUDACC__ -#define CUDAQX_RPC_HD __host__ __device__ -#else -#define CUDAQX_RPC_HD -#endif - -CUDAQX_RPC_HD constexpr std::size_t bit_packed_bytes(std::size_t num_bits) { +constexpr std::size_t bit_packed_bytes(std::size_t num_bits) { return (num_bits + 7) / 8; } -CUDAQX_RPC_HD constexpr std::size_t align_to_8(std::size_t bytes) { +constexpr std::size_t align_to_8(std::size_t bytes) { return (bytes + 7) & ~static_cast(7); } -#undef CUDAQX_RPC_HD - -// Maximum number of decoders the realtime dispatch can register at once. GPU -// dispatch implementations use it to size their device-side state tables; host -// session code uses it for early range checks without depending on private GPU -// dispatch headers. -inline constexpr std::size_t kMaxDispatchedDecoders = 32; - } // namespace cudaq::qec::decoding::rpc diff --git a/libs/qec/include/cudaq/qec/realtime/decoding_config.h b/libs/qec/include/cudaq/qec/realtime/decoding_config.h index ca98918c9..ec1f63e56 100644 --- a/libs/qec/include/cudaq/qec/realtime/decoding_config.h +++ b/libs/qec/include/cudaq/qec/realtime/decoding_config.h @@ -10,53 +10,166 @@ #include "cuda-qx/core/heterogeneous_map.h" #include -#include #include #include +#include #include namespace cudaq::qec::decoding::config { -/// Transport type for a decoder session. -/// cpu_roce: CpuRoceTransceiver / SoftRoCE (dev, CI, no GPU required) -/// gpu_roce: GpuRoceTransceiver / DOCA (production, real ConnectX) -enum class DecoderTransport { cpu_roce, gpu_roce }; - -/// @brief Decoder-specific constructor arguments, stored as a -/// `cudaqx::heterogeneous_map` -- the form every decoder's constructor -/// consumes. YAML conversion and key validation are driven by the parameter -/// schema the decoder registered (see cudaq/qec/decoder_config_schema.h), so -/// out-of-tree decoders participate without any framework changes. -class decoder_custom_args_t { -public: - decoder_custom_args_t() = default; - decoder_custom_args_t(const cudaqx::heterogeneous_map &m) : map_(m) {} +struct srelay_bp_config { + std::optional pre_iter; + std::optional num_sets; + std::optional stopping_criterion; + std::optional stop_nconv; - decoder_custom_args_t &operator=(const cudaqx::heterogeneous_map &m) { - map_ = m; - return *this; - } + bool operator==(const srelay_bp_config &) const = default; + + __attribute__((visibility("default"))) cudaqx::heterogeneous_map + to_heterogeneous_map() const; + + __attribute__((visibility("default"))) static srelay_bp_config + from_heterogeneous_map(const cudaqx::heterogeneous_map &map); +}; + +struct nv_qldpc_decoder_config { + std::optional use_sparsity; + std::optional error_rate; + std::optional> error_rate_vec; + std::optional max_iterations; + std::optional n_threads; + std::optional use_osd; + std::optional osd_method; + std::optional osd_order; + std::optional bp_batch_size; + std::optional osd_batch_size; + std::optional iter_per_check; + std::optional clip_value; + std::optional bp_method; + std::optional scale_factor; + std::optional proc_float; + std::optional gamma0; + std::optional> gamma_dist; + std::optional>> explicit_gammas; + std::optional srelay_config; + std::optional bp_seed; + std::optional composition; + + bool operator==(const nv_qldpc_decoder_config &) const = default; + // opt_results is currently not supported for real-time decoding. + + __attribute__((visibility("default"))) cudaqx::heterogeneous_map + to_heterogeneous_map() const; + + __attribute__((visibility("default"))) static nv_qldpc_decoder_config + from_heterogeneous_map(const cudaqx::heterogeneous_map &map); +}; + +struct multi_error_lut_config { + std::optional lut_error_depth; + // error_rate_vec is currently not supported for real-time decoding. + // opt_results is currently not supported for real-time decoding. + + bool operator==(const multi_error_lut_config &) const = default; + + __attribute__((visibility("default"))) cudaqx::heterogeneous_map + to_heterogeneous_map() const; + + __attribute__((visibility("default"))) static multi_error_lut_config + from_heterogeneous_map(const cudaqx::heterogeneous_map &map); +}; + +struct single_error_lut_config { + bool operator==(const single_error_lut_config &) const = default; + + __attribute__((visibility("default"))) cudaqx::heterogeneous_map + to_heterogeneous_map() const; + + __attribute__((visibility("default"))) static single_error_lut_config + from_heterogeneous_map(const cudaqx::heterogeneous_map &map); +}; + +struct pymatching_config { + std::optional> error_rate_vec; + std::optional merge_strategy; + + bool operator==(const pymatching_config &) const = default; - cudaqx::heterogeneous_map &map() { return map_; } - const cudaqx::heterogeneous_map &map() const { return map_; } - bool empty() const { return map_.empty(); } + __attribute__((visibility("default"))) cudaqx::heterogeneous_map + to_heterogeneous_map() const; + + __attribute__((visibility("default"))) static pymatching_config + from_heterogeneous_map(const cudaqx::heterogeneous_map &map); +}; + +struct chromobius_config { + std::optional drop_mobius_errors_involving_remnant_errors; + std::optional ignore_decomposition_failures; + std::optional include_coords_in_mobius_dem; + std::optional return_weight; + std::optional write_mobius_match_to_stderr; + + bool operator==(const chromobius_config &) const = default; + + __attribute__((visibility("default"))) cudaqx::heterogeneous_map + to_heterogeneous_map() const; - /// Deep equality over the canonical custom-args value kinds. - __attribute__((visibility("default"))) bool - operator==(const decoder_custom_args_t &other) const; + __attribute__((visibility("default"))) static chromobius_config + from_heterogeneous_map(const cudaqx::heterogeneous_map &map); +}; + +using global_decoder_config = + std::variant; + +struct trt_decoder_config { + std::optional onnx_load_path; + std::optional engine_load_path; + std::optional engine_save_path; + std::optional precision; + std::optional memory_workspace; + std::optional batch_size; + std::optional use_cuda_graph; + std::optional global_decoder; + global_decoder_config global_decoder_params; + + bool operator==(const trt_decoder_config &) const = default; + + __attribute__((visibility("default"))) cudaqx::heterogeneous_map + to_heterogeneous_map() const; -private: - cudaqx::heterogeneous_map map_; + __attribute__((visibility("default"))) static trt_decoder_config + from_heterogeneous_map(const cudaqx::heterogeneous_map &map); +}; + +struct sliding_window_config { + std::optional window_size; + std::optional step_size; + std::optional num_syndromes_per_round; + std::optional num_boundary_syndromes; + std::optional straddle_start_round; + std::optional straddle_end_round; + std::vector error_rate_vec; + std::string inner_decoder_name; + + // Concrete inner decoder configurations (only one should be set based on + // inner_decoder_name) + std::optional single_error_lut_params; + std::optional multi_error_lut_params; + std::optional nv_qldpc_decoder_params; + + bool operator==(const sliding_window_config &) const = default; + + __attribute__((visibility("default"))) cudaqx::heterogeneous_map + to_heterogeneous_map() const; + + __attribute__((visibility("default"))) static sliding_window_config + from_heterogeneous_map(const cudaqx::heterogeneous_map &map); }; /// @brief Configuration structure for decoder options. struct decoder_config { int64_t id = 0; std::string type; - /// Transport used to receive syndromes and send corrections for this decoder. - /// Defaults to cpu_roce. Set to gpu_roce for decoders where syndrome bits - /// are DMA'd directly to GPU VRAM (e.g. nv_qldpc_decoder with RelayBP). - DecoderTransport transport = DecoderTransport::cpu_roce; /// CUDA device this decoder is pinned to at construction (see the /// "cuda_device_id" decoder parameter). Placement knob common to any /// GPU-accelerated decoder, hence at this level rather than inside the @@ -67,24 +180,60 @@ struct decoder_config { std::vector H_sparse; std::vector O_sparse; std::vector D_sparse; - decoder_custom_args_t decoder_custom_args; + std::variant + decoder_custom_args; bool operator==(const decoder_config &) const = default; - /// Return the parameter map a decoder's constructor should receive: the - /// stored custom args with schema-declared defaults materialized (see - /// materialize_default_args in cudaq/qec/decoder_config_schema.h) when a - /// schema is registered for `type`, so programmatically built configs get - /// the same defaulting the YAML parse path applies. __attribute__((visibility("default"))) cudaqx::heterogeneous_map - decoder_custom_args_to_heterogeneous_map() const; + decoder_custom_args_to_heterogeneous_map() const { + if (std::holds_alternative(decoder_custom_args)) { + return std::get(decoder_custom_args) + .to_heterogeneous_map(); + } else if (std::holds_alternative( + decoder_custom_args)) { + return std::get(decoder_custom_args) + .to_heterogeneous_map(); + } else if (std::holds_alternative( + decoder_custom_args)) { + return std::get(decoder_custom_args) + .to_heterogeneous_map(); + } else if (std::holds_alternative( + decoder_custom_args)) { + return std::get(decoder_custom_args) + .to_heterogeneous_map(); + } else if (std::holds_alternative( + decoder_custom_args)) { + return std::get(decoder_custom_args) + .to_heterogeneous_map(); + } else if (std::holds_alternative(decoder_custom_args)) { + return std::get(decoder_custom_args) + .to_heterogeneous_map(); + } + return cudaqx::heterogeneous_map(); + } - /// Validate `decoder_custom_args` against the parameter schema registered - /// for `type`: unknown keys, missing required keys, and the schema's own - /// validate hook (if any). Throws std::runtime_error on the first - /// violation. YAML parsing applies the same checks automatically; call this - /// to vet a configuration built programmatically before using it. - __attribute__((visibility("default"))) void validate_custom_args() const; + __attribute__((visibility("default"))) void + set_decoder_custom_args_from_heterogeneous_map( + const cudaqx::heterogeneous_map &map) { + if (type == "single_error_lut") { + decoder_custom_args = + single_error_lut_config::from_heterogeneous_map(map); + } else if (type == "multi_error_lut") { + decoder_custom_args = multi_error_lut_config::from_heterogeneous_map(map); + } else if (type == "nv-qldpc-decoder") { + decoder_custom_args = + nv_qldpc_decoder_config::from_heterogeneous_map(map); + } else if (type == "sliding_window") { + decoder_custom_args = sliding_window_config::from_heterogeneous_map(map); + } else if (type == "trt_decoder") { + decoder_custom_args = trt_decoder_config::from_heterogeneous_map(map); + } else if (type == "pymatching") { + decoder_custom_args = pymatching_config::from_heterogeneous_map(map); + } + } __attribute__((visibility("default"))) std::string to_yaml_str(int column_wrap = 80); @@ -98,27 +247,12 @@ class multi_decoder_config { bool operator==(const multi_decoder_config &) const = default; - /// Validate every decoder's custom args (see - /// decoder_config::validate_custom_args). - __attribute__((visibility("default"))) void validate_custom_args() const; - __attribute__((visibility("default"))) std::string to_yaml_str(int column_wrap = 80); __attribute__((visibility("default"))) static multi_decoder_config from_yaml_str(const std::string_view yaml_str); }; -/// @brief Generate a JSON Schema (draft 2020-12) document describing valid -/// `multi_decoder_config` YAML files, so third-party tools (check-jsonschema, -/// python jsonschema, yaml-language-server, ...) can validate user-provided -/// configurations offline. The per-decoder `decoder_custom_args` sections are -/// generated from the parameter schemas registered at call time, so decoder -/// plugins (in-tree and out-of-tree alike) appear automatically once their -/// library is loaded. Schema `validate` hooks are arbitrary C++ and are not -/// representable; a document that passes the JSON Schema may still be -/// rejected by those hooks when parsed. -__attribute__((visibility("default"))) std::string decoder_config_json_schema(); - /// @brief Configure the decoders (`multi_decoder_config` variant). This /// function configures both local decoders, and if running on remote target /// hardware, will submit the configuration to the remote target for further @@ -147,15 +281,4 @@ configure_decoders_from_str(const char *config_str); /// @brief Finalize the decoders. This function finalizes local decoders. __attribute__((visibility("default"))) void finalize_decoders(); -/// @brief Return the most recently passed multi_decoder_config, or an empty -/// pointer if configure_decoders() has not been called in this process. -/// Used by the decoding-server DeviceCallService plugin to build -/// DecodingSessions on the in-process host_dispatch path without requiring -/// CUDAQ_QEC_DECODER_CONFIG. Returns shared ownership: a concurrent -/// configure_decoders() replaces the stored config but cannot free it out -/// from under the caller. -__attribute__((visibility("default"))) -std::shared_ptr -last_configured_multi_decoder_config(); - } // namespace cudaq::qec::decoding::config diff --git a/libs/qec/include/cudaq/qec/realtime/gpu_kernels.cuh b/libs/qec/include/cudaq/qec/realtime/gpu_kernels.cuh index b3b547946..3038f8ced 100644 --- a/libs/qec/include/cudaq/qec/realtime/gpu_kernels.cuh +++ b/libs/qec/include/cudaq/qec/realtime/gpu_kernels.cuh @@ -49,19 +49,13 @@ __device__ void preprocess_detector(std::size_t detector_idx, /// For observable at row `observable_idx`: /// 1. Read soft decisions at column positions from O matrix row /// 2. Threshold each value (>= thresh -> 1, else 0) -/// 3. XOR the hard decisions to compute this window's correction -/// 4. XOR-accumulate that correction into corrections[observable_idx] -/// -/// XOR-accumulate semantics mirror the legacy CPU contract where every -/// full-window decode XORs that window's observable bits into a per-shot -/// corrections buffer. Callers are responsible for zeroing `corrections` at -/// shot boundaries. +/// 3. XOR the hard decisions to compute correction /// /// @param observable_idx Index of the observable to compute /// @param soft_decisions Input soft decisions from decoder /// @param O_row_ptr CSR row pointers for O matrix /// @param O_col_idx CSR column indices for O matrix -/// @param corrections In/out per-shot accumulator +/// @param corrections Output corrections array /// @param num_observables Total number of observables /// @param thresh Soft-to-hard threshold (default 0.5) __device__ void postprocess_observable( @@ -88,12 +82,11 @@ __device__ void preprocess_all(const uint8_t *__restrict__ measurements, /// @brief Postprocess all observables using grid-stride loop. /// /// All threads in the grid participate, each handling a subset of observables. -/// Calls postprocess_observable per observable with XOR-accumulate semantics. /// /// @param soft_decisions Input soft decoder output /// @param O_row_ptr CSR row pointers for O matrix (size: num_observables + 1) /// @param O_col_idx CSR column indices for O matrix -/// @param corrections In/out per-shot accumulator +/// @param corrections Output observable corrections /// @param num_observables Number of observables (rows in O matrix) /// @param thresh Threshold for soft-to-hard conversion (default 0.5) __device__ void postprocess_all(const float_t *__restrict__ soft_decisions, diff --git a/libs/qec/lib/CMakeLists.txt b/libs/qec/lib/CMakeLists.txt index 37842f4be..0a8fad91e 100644 --- a/libs/qec/lib/CMakeLists.txt +++ b/libs/qec/lib/CMakeLists.txt @@ -42,7 +42,6 @@ endif() set(DECODERS_SOURCES decoder.cpp decoder_config_payload.cpp - decoder_config_schema.cpp detector_error_model.cpp logger.cpp logger_forwarder.cpp diff --git a/libs/qec/lib/decoder_config_schema.cpp b/libs/qec/lib/decoder_config_schema.cpp deleted file mode 100644 index f3711ba32..000000000 --- a/libs/qec/lib/decoder_config_schema.cpp +++ /dev/null @@ -1,473 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2024 - 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#include "cudaq/qec/decoder_config_schema.h" -#include "cuda-qx/core/tuple_utils.h" -#include "cuda-qx/core/type_traits.h" -#include "cudaq/qec/logger.h" -#include -#include -#include -#include -#include - -namespace cudaq::qec::decoding::config { - -namespace { - -// Node-based map: pointers returned by find_decoder_schema stay valid across -// later registrations. Schemas are never erased or unregistered -- including -// when the plugin that registered one is dlclose'd (cleanup_decoder_plugins -// runs from a library destructor at exit). The registry is intentionally -// leaked, like get_plugin_handles() and INSTANTIATE_REGISTRY: destroying it -// at static-destruction time could run std::function (validate hook) -// destructors whose code lives in an already-unloaded plugin. -std::map &schema_registry() { - static auto *registry = new std::map(); - return *registry; -} - -std::mutex &schema_registry_mutex() { - static auto *m = new std::mutex(); - return *m; -} - -} // namespace - -void register_decoder_schema(decoder_schema schema) { - std::lock_guard lock(schema_registry_mutex()); - auto name = schema.name; - schema_registry().insert_or_assign(std::move(name), std::move(schema)); -} - -const decoder_schema *find_decoder_schema(const std::string &name) { - std::lock_guard lock(schema_registry_mutex()); - auto ®istry = schema_registry(); - auto iter = registry.find(name); - if (iter == registry.end()) - return nullptr; - return &iter->second; -} - -std::vector registered_decoder_schema_names() { - std::lock_guard lock(schema_registry_mutex()); - std::vector names; - for (const auto &[name, schema] : schema_registry()) - names.push_back(name); - return names; -} - -// --------------------------------------------------------------------------- -// Schema validation of programmatically built custom-args maps -// --------------------------------------------------------------------------- - -namespace { - -// True when `value` would be readable as T by heterogeneous_map::get -- -// i.e. it holds T or one of T's RelatedTypesMap entries (the same tuple -// get iterates). Pointer-based probing so validation never copies a value. -template -bool value_readable_as(const std::any &value) { - if (std::any_cast(&value)) - return true; - bool readable = false; - cudaqx::tuple_for_each( - typename cudaqx::RelatedTypesMap::types(), [&](auto &&el) { - readable = readable || - std::any_cast< - std::remove_cv_t>>( - &value); - }); - return readable; -} - -// True when `value` would be readable as `kind`'s canonical storage type. -bool value_matches_kind(param_kind kind, const std::any &value) { - switch (kind) { - case param_kind::boolean: - return value_readable_as(value); - case param_kind::int32: - return value_readable_as(value); - case param_kind::uint64: - return value_readable_as(value); - case param_kind::f64: - return value_readable_as(value); - case param_kind::string: - return value_readable_as(value); - case param_kind::f64_vec: - return value_readable_as>(value); - case param_kind::f64_matrix: - return value_readable_as>>(value); - case param_kind::subschema: - case param_kind::discriminated: - return value_readable_as(value); - } - return false; -} - -const char *kind_description(param_kind kind) { - switch (kind) { - case param_kind::boolean: - return "boolean"; - case param_kind::int32: - return "32-bit int"; - case param_kind::uint64: - return "non-negative int"; - case param_kind::f64: - return "float"; - case param_kind::string: - return "string"; - case param_kind::f64_vec: - return "list-of-float"; - case param_kind::f64_matrix: - return "list-of-list-of-float"; - case param_kind::subschema: - case param_kind::discriminated: - return "mapping"; - } - return "unknown"; -} - -} // namespace - -void validate_custom_args(const decoder_schema &schema, - const cudaqx::heterogeneous_map &args, - const std::string &context) { - for (const auto &kv : args) { - const std::string &key = kv.first; - const param_spec *spec = nullptr; - for (const auto &candidate : schema.params) { - if (candidate.key == key) { - spec = &candidate; - break; - } - } - if (!spec) - throw std::runtime_error("Unknown key '" + key + "' in " + context + "."); - - // Every value must be readable as its kind's canonical storage type; - // this is what YAML emission and decoder construction will do with it, - // so a map that validates is guaranteed to serialize (round-trip - // invariant). Values that arrived through the YAML parser or the typed - // Python setter always pass; this catches maps built by other means - // (e.g. a Python dict assigned before `type`, stored generically). - if (!value_matches_kind(spec->kind, kv.second)) - throw std::runtime_error( - "Key '" + key + "' in " + context + " does not hold a " + - kind_description(spec->kind) + " value as its schema kind requires."); - - if (spec->kind == param_kind::subschema) { - const auto *nested_schema = find_decoder_schema(spec->subschema); - if (!nested_schema) - throw std::runtime_error("No schema registered under '" + - spec->subschema + "' (needed to validate '" + - key + "')."); - validate_custom_args( - *nested_schema, *std::any_cast(&kv.second), - context + "." + key); - } else if (spec->kind == param_kind::discriminated) { - std::string discriminator_value; - if (args.contains(spec->discriminator)) - discriminator_value = args.get(spec->discriminator); - if (discriminator_value.empty()) - throw std::runtime_error("'" + key + "' is present but '" + - spec->discriminator + "' is not set in " + - context + "."); - const auto *nested_schema = find_decoder_schema(discriminator_value); - if (!nested_schema) - throw std::runtime_error( - "'" + key + "' does not support " + spec->discriminator + " '" + - discriminator_value + - "': no parameter schema is registered under that name."); - validate_custom_args( - *nested_schema, *std::any_cast(&kv.second), - context + "." + key); - } - } - for (const auto &spec : schema.params) - if (spec.required && !args.contains(spec.key)) - throw std::runtime_error("Missing required key '" + spec.key + "' in " + - context + "."); - if (schema.validate) - schema.validate(args); -} - -void validate_custom_args(const std::string &schema_name, - const cudaqx::heterogeneous_map &args) { - const auto *schema = find_decoder_schema(schema_name); - if (!schema) { - if (args.empty()) - return; - throw std::runtime_error( - "Decoder type '" + schema_name + - "' has no registered parameter schema; its decoder_custom_args " - "cannot be validated (or serialized to YAML). Register a schema from " - "the decoder's plugin library with register_decoder_schema()."); - } - validate_custom_args(*schema, args, "'" + schema_name + "' parameters"); -} - -void materialize_default_args(const decoder_schema &schema, - cudaqx::heterogeneous_map &args) { - for (const auto &spec : schema.params) { - const decoder_schema *nested_schema = nullptr; - if (spec.kind == param_kind::discriminated) { - std::string discriminator_value; - if (args.contains(spec.discriminator)) - discriminator_value = args.get(spec.discriminator); - nested_schema = discriminator_value.empty() - ? nullptr - : find_decoder_schema(discriminator_value); - if (spec.materialize_empty && nested_schema && !args.contains(spec.key)) - args.insert(spec.key, cudaqx::heterogeneous_map()); - } else if (spec.kind == param_kind::subschema) { - nested_schema = find_decoder_schema(spec.subschema); - } - if (nested_schema && args.contains(spec.key)) { - auto nested = args.get(spec.key); - materialize_default_args(*nested_schema, nested); - args.insert(spec.key, nested); - } - } -} - -void drop_non_schema_keys(const decoder_schema &schema, - cudaqx::heterogeneous_map &args) { - // Single pass: recurse into nested sections in place (no copies), and note - // whether any key at this level is unknown. Only when one is does the map - // get rebuilt -- the common all-keys-known case leaves `args` untouched. - bool has_unknown = false; - for (auto &kv : args) { - const param_spec *spec = nullptr; - for (const auto &candidate : schema.params) { - if (candidate.key == kv.first) { - spec = &candidate; - break; - } - } - if (!spec) { - CUDA_QEC_WARN("Key '{}' is not in the '{}' parameter schema; it is " - "excluded from the decoder parameters and from emitted " - "YAML.", - kv.first, schema.name); - has_unknown = true; - continue; - } - const decoder_schema *nested_schema = nullptr; - if (spec->kind == param_kind::subschema) { - nested_schema = find_decoder_schema(spec->subschema); - } else if (spec->kind == param_kind::discriminated && - args.contains(spec->discriminator)) { - nested_schema = - find_decoder_schema(args.get(spec->discriminator)); - } - if (nested_schema) - if (auto *nested = std::any_cast(&kv.second)) - drop_non_schema_keys(*nested_schema, *nested); - } - if (!has_unknown) - return; - cudaqx::heterogeneous_map kept; - for (const auto &kv : args) { - for (const auto &spec : schema.params) { - if (spec.key == kv.first) { - kept.insert(kv.first, kv.second); - break; - } - } - } - args = std::move(kept); -} - -// --------------------------------------------------------------------------- -// Deep equality over canonical custom-args value kinds -// --------------------------------------------------------------------------- - -namespace { - -// Sign-aware integer representation so unsigned values above 2^63 never -// alias negative signed values (a plain long long cast would wrap -// size_t(2^64-1) to -1 and report it equal to int(-1)). -struct integer_value { - bool negative = false; - unsigned long long magnitude = 0; - bool operator==(const integer_value &other) const { - return negative == other.negative && magnitude == other.magnitude; - } - double as_double() const { - double d = static_cast(magnitude); - return negative ? -d : d; - } -}; - -std::optional as_integer(const std::any &v) { - auto from_signed = [](long long s) { - integer_value iv; - iv.negative = s < 0; - iv.magnitude = iv.negative ? ~static_cast(s) + 1ULL - : static_cast(s); - return iv; - }; - auto from_unsigned = [](unsigned long long u) { - return integer_value{false, u}; - }; - if (auto *p = std::any_cast(&v)) - return from_signed(*p); - if (auto *p = std::any_cast(&v)) - return from_signed(*p); - if (auto *p = std::any_cast(&v)) - return from_signed(*p); - if (auto *p = std::any_cast(&v)) - return from_signed(*p); - if (auto *p = std::any_cast(&v)) - return from_unsigned(*p); - if (auto *p = std::any_cast(&v)) - return from_unsigned(*p); - if (auto *p = std::any_cast(&v)) - return from_unsigned(*p); - if (auto *p = std::any_cast(&v)) - return from_unsigned(*p); - return std::nullopt; -} - -std::optional as_floating(const std::any &v) { - if (auto *p = std::any_cast(&v)) - return *p; - if (auto *p = std::any_cast(&v)) - return static_cast(*p); - return std::nullopt; -} - -bool any_values_equal(const std::any &a, const std::any &b) { - // bool is kept distinct from the integer family so `true` never silently - // matches `1` from a differently-built configuration. - if (auto *pa = std::any_cast(&a)) { - auto *pb = std::any_cast(&b); - return pb && *pa == *pb; - } - if (std::any_cast(&b)) - return false; - - if (auto ia = as_integer(a)) { - if (auto ib = as_integer(b)) - return *ia == *ib; - if (auto fb = as_floating(b)) - return ia->as_double() == *fb; - return false; - } - if (auto fa = as_floating(a)) { - if (auto fb = as_floating(b)) - return *fa == *fb; - if (auto ib = as_integer(b)) - return *fa == ib->as_double(); - return false; - } - if (auto *pa = std::any_cast(&a)) { - auto *pb = std::any_cast(&b); - return pb && *pa == *pb; - } - if (auto *pa = std::any_cast>(&a)) { - auto *pb = std::any_cast>(&b); - return pb && *pa == *pb; - } - if (auto *pa = std::any_cast>>(&a)) { - auto *pb = std::any_cast>>(&b); - return pb && *pa == *pb; - } - if (auto *pa = std::any_cast(&a)) { - auto *pb = std::any_cast(&b); - return pb && custom_args_maps_equal(*pa, *pb); - } - // Unknown value type: conservatively unequal. - return false; -} - -} // namespace - -bool custom_args_maps_equal(const cudaqx::heterogeneous_map &a, - const cudaqx::heterogeneous_map &b) { - if (a.size() != b.size()) - return false; - - for (const auto &[key_a, val_a] : a) { - bool found = false; - for (const auto &[key_b, val_b] : b) { - if (key_a != key_b) - continue; - found = true; - if (!any_values_equal(val_a, val_b)) - return false; - break; - } - if (!found) - return false; - } - return true; -} - -// --------------------------------------------------------------------------- -// Hosted decoder schemas -// -// Decoder schemas are registered by the shared library that ships the -// decoder: lut.cpp and sliding_window.cpp register theirs in this library, -// and the pymatching / chromobius / trt_decoder plugins register theirs from -// their own .so. The nv-qldpc-decoder schema is hosted here temporarily: the -// decoder is a proprietary out-of-tree plugin, and its schema (plus the -// srelay_bp subschema it references) moves into that plugin once it links -// against this registry. Third-party decoders register their schema from -// their own plugin library instead of editing this file. -// --------------------------------------------------------------------------- - -namespace { - -struct hosted_schema_registrar { - hosted_schema_registrar() { - using k = param_kind; - - register_decoder_schema({"srelay_bp", - { - {"pre_iter", k::uint64}, - {"num_sets", k::uint64}, - {"stopping_criterion", k::string}, - {"stop_nconv", k::uint64}, - }}); - - register_decoder_schema( - {"nv-qldpc-decoder", - { - {"use_sparsity", k::boolean}, - {"error_rate", k::f64}, - {"error_rate_vec", k::f64_vec}, - {"max_iterations", k::int32}, - {"n_threads", k::int32}, - {"use_osd", k::boolean}, - {"osd_method", k::int32}, - {"osd_order", k::int32}, - {"bp_batch_size", k::int32}, - {"osd_batch_size", k::int32}, - {"iter_per_check", k::int32}, - {"clip_value", k::f64}, - {"bp_method", k::int32}, - {"scale_factor", k::f64}, - {"proc_float", k::string}, - {"gamma0", k::f64}, - {"gamma_dist", k::f64_vec}, - {"explicit_gammas", k::f64_matrix}, - {"bp_seed", k::int32}, - {"srelay_config", k::subschema, false, "srelay_bp"}, - {"composition", k::int32}, - {"repeatable", k::boolean}, - }}); - } -}; - -hosted_schema_registrar hosted_schemas; - -} // namespace - -} // namespace cudaq::qec::decoding::config diff --git a/libs/qec/lib/decoders/lut.cpp b/libs/qec/lib/decoders/lut.cpp index a25f4bdff..1f49803a8 100644 --- a/libs/qec/lib/decoders/lut.cpp +++ b/libs/qec/lib/decoders/lut.cpp @@ -7,7 +7,6 @@ ******************************************************************************/ #include "cudaq/qec/decoder.h" -#include "cudaq/qec/decoder_config_schema.h" #include "cudaq/qec/logger.h" #include #include @@ -256,21 +255,4 @@ class single_error_lut : public multi_error_lut { CUDAQ_EXT_PT_REGISTER_TYPE(single_error_lut) -// Parameter schemas for the realtime decoding YAML (`decoder_custom_args`), -// registered alongside the decoders they describe. -namespace { -struct lut_schema_registrar { - lut_schema_registrar() { - using k = decoding::config::param_kind; - decoding::config::register_decoder_schema({"single_error_lut", {}}); - decoding::config::register_decoder_schema( - {"multi_error_lut", - { - {"lut_error_depth", k::int32}, - }}); - } -}; -lut_schema_registrar register_lut_schemas; -} // namespace - } // namespace cudaq::qec diff --git a/libs/qec/lib/decoders/plugins/chromobius/chromobius.cpp b/libs/qec/lib/decoders/plugins/chromobius/chromobius.cpp index 26877e1d6..54712c57d 100644 --- a/libs/qec/lib/decoders/plugins/chromobius/chromobius.cpp +++ b/libs/qec/lib/decoders/plugins/chromobius/chromobius.cpp @@ -9,7 +9,6 @@ #include "chromobius/decode/decoder.h" #include "stim.h" #include "cudaq/qec/decoder.h" -#include "cudaq/qec/decoder_config_schema.h" #include #include @@ -190,26 +189,4 @@ class chromobius : public decoder { CUDAQ_EXT_PT_REGISTER_TYPE(chromobius) -// Parameter schema for the realtime decoding YAML (`decoder_custom_args` for -// `type: chromobius`, and the trt_decoder `global_decoder_params` section when -// `global_decoder: chromobius`). Registered here so the schema ships with the -// decoder itself. -namespace { -struct chromobius_schema_registrar { - chromobius_schema_registrar() { - using k = cudaq::qec::decoding::config::param_kind; - cudaq::qec::decoding::config::register_decoder_schema( - {"chromobius", - { - {"drop_mobius_errors_involving_remnant_errors", k::boolean}, - {"ignore_decomposition_failures", k::boolean}, - {"include_coords_in_mobius_dem", k::boolean}, - {"return_weight", k::boolean}, - {"write_mobius_match_to_stderr", k::boolean}, - }}); - } -}; -chromobius_schema_registrar register_chromobius_schema; -} // namespace - } // namespace cudaq::qec diff --git a/libs/qec/lib/decoders/plugins/example/single_error_lut_example.cpp b/libs/qec/lib/decoders/plugins/example/single_error_lut_example.cpp index 661306d80..861f65771 100644 --- a/libs/qec/lib/decoders/plugins/example/single_error_lut_example.cpp +++ b/libs/qec/lib/decoders/plugins/example/single_error_lut_example.cpp @@ -7,7 +7,6 @@ ******************************************************************************/ #include "cudaq/qec/decoder.h" -#include "cudaq/qec/decoder_config_schema.h" #include #include #include @@ -86,20 +85,4 @@ class single_error_lut_example : public decoder { CUDAQ_EXT_PT_REGISTER_TYPE(single_error_lut_example) -// Registering a parameter schema alongside the decoder makes the decoder -// configurable through the realtime decoding YAML (`decoder_custom_args`) -// without any changes to the cudaq-qec libraries: the YAML layer converts the -// section to the heterogeneous_map this decoder's constructor receives, using -// only this description. This decoder takes no parameters, so the schema is -// empty; list each accepted key and its param_kind here otherwise. -namespace { -struct schema_registrar { - schema_registrar() { - cudaq::qec::decoding::config::register_decoder_schema( - {"single_error_lut_example", {}}); - } -}; -schema_registrar register_schema; -} // namespace - } // namespace cudaq::qec diff --git a/libs/qec/lib/decoders/plugins/pymatching/pymatching.cpp b/libs/qec/lib/decoders/plugins/pymatching/pymatching.cpp index f1e26d8e2..08fbd399c 100644 --- a/libs/qec/lib/decoders/plugins/pymatching/pymatching.cpp +++ b/libs/qec/lib/decoders/plugins/pymatching/pymatching.cpp @@ -9,7 +9,6 @@ #include "pymatching/sparse_blossom/driver/mwpm_decoding.h" #include "pymatching/sparse_blossom/driver/user_graph.h" #include "cudaq/qec/decoder.h" -#include "cudaq/qec/decoder_config_schema.h" #include #include #include @@ -267,23 +266,4 @@ class pymatching : public decoder { CUDAQ_EXT_PT_REGISTER_TYPE(pymatching) -// Parameter schema for the realtime decoding YAML (`decoder_custom_args` for -// `type: pymatching`, and the trt_decoder `global_decoder_params` section when -// `global_decoder: pymatching`). Registered here so the schema ships with the -// decoder itself. -namespace { -struct pymatching_schema_registrar { - pymatching_schema_registrar() { - using k = cudaq::qec::decoding::config::param_kind; - cudaq::qec::decoding::config::register_decoder_schema( - {"pymatching", - { - {"error_rate_vec", k::f64_vec}, - {"merge_strategy", k::string}, - }}); - } -}; -pymatching_schema_registrar register_pymatching_schema; -} // namespace - } // namespace cudaq::qec diff --git a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp index 675b25fd9..58e093a7d 100644 --- a/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp +++ b/libs/qec/lib/decoders/plugins/trt_decoder/trt_decoder.cpp @@ -7,7 +7,6 @@ ******************************************************************************/ #include "cudaq/qec/decoder.h" -#include "cudaq/qec/decoder_config_schema.h" #include "cudaq/qec/logger.h" #include "cudaq/qec/pcm_utils.h" #include "cudaq/qec/trt_decoder_internal.h" @@ -1089,35 +1088,6 @@ void trt_decoder::check_cuda() { CUDAQ_EXT_PT_REGISTER_TYPE(trt_decoder) -// Parameter schema for the realtime decoding YAML (`decoder_custom_args` for -// `type: trt_decoder`). `global_decoder_params` is a discriminated section: -// it is parsed with the schema registered under the value of -// `global_decoder` (e.g. "pymatching" or "chromobius", each registered by its -// own plugin), and an empty section is materialized when `global_decoder` -// names a registered schema but no params are given. -namespace { -struct trt_decoder_schema_registrar { - trt_decoder_schema_registrar() { - using k = cudaq::qec::decoding::config::param_kind; - cudaq::qec::decoding::config::register_decoder_schema( - {"trt_decoder", - { - {"onnx_load_path", k::string}, - {"engine_load_path", k::string}, - {"engine_save_path", k::string}, - {"precision", k::string}, - {"memory_workspace", k::uint64}, - {"batch_size", k::uint64}, - {"use_cuda_graph", k::boolean}, - {"global_decoder", k::string}, - {"global_decoder_params", k::discriminated, false, "", - "global_decoder", /*materialize_empty=*/true}, - }}); - } -}; -trt_decoder_schema_registrar register_trt_decoder_schema; -} // namespace - } // namespace cudaq::qec namespace cudaq::qec::trt_decoder_internal { diff --git a/libs/qec/lib/decoders/sliding_window.cpp b/libs/qec/lib/decoders/sliding_window.cpp index 8795c1bc4..adae88a4b 100644 --- a/libs/qec/lib/decoders/sliding_window.cpp +++ b/libs/qec/lib/decoders/sliding_window.cpp @@ -7,7 +7,6 @@ ******************************************************************************/ #include "sliding_window.h" -#include "cudaq/qec/decoder_config_schema.h" #include "cudaq/qec/logger.h" #include "cudaq/qec/pcm_utils.h" #include @@ -401,61 +400,4 @@ std::size_t sliding_window::get_num_detector_layers() const { CUDAQ_EXT_PT_REGISTER_TYPE(sliding_window) -// Parameter schema for the realtime decoding YAML (`decoder_custom_args` for -// `type: sliding_window`). `inner_decoder_params` is a discriminated section -// parsed with the schema registered under the value of `inner_decoder_name` -// (whichever decoder that names must have registered its own schema). -// Unknown-key and required-key checks are applied by the framework from the -// param specs alone; the `validate` hook adds the cross-field constraints -// those specs cannot express. -namespace { -struct sliding_window_schema_registrar { - sliding_window_schema_registrar() { - using k = decoding::config::param_kind; - decoding::config::decoder_schema schema{ - "sliding_window", - { - {"window_size", k::uint64}, - {"step_size", k::uint64}, - {"num_syndromes_per_round", k::uint64}, - {"num_boundary_syndromes", k::uint64}, - {"straddle_start_round", k::boolean}, - {"straddle_end_round", k::boolean}, - {"error_rate_vec", k::f64_vec, /*required=*/true}, - {"inner_decoder_name", k::string, /*required=*/true}, - {"inner_decoder_params", k::discriminated, false, "", - "inner_decoder_name", /*materialize_empty=*/false}, - }}; - schema.validate = [](const cudaqx::heterogeneous_map &args) { - if (args.contains("window_size") && args.contains("step_size")) { - auto window_size = args.get("window_size"); - auto step_size = args.get("step_size"); - if (step_size < 1 || step_size > window_size) - throw std::runtime_error(fmt::format( - "sliding_window parameters: step_size ({}) must be between 1 " - "and window_size ({})", - step_size, window_size)); - } - if (args.contains("num_boundary_syndromes") && - args.contains("num_syndromes_per_round")) { - auto num_boundary_syndromes = - args.get("num_boundary_syndromes"); - auto num_syndromes_per_round = - args.get("num_syndromes_per_round"); - if (num_boundary_syndromes > num_syndromes_per_round) - throw std::runtime_error(fmt::format( - "sliding_window parameters: num_boundary_syndromes ({}) must be " - "<= num_syndromes_per_round ({})", - num_boundary_syndromes, num_syndromes_per_round)); - } - if (args.get>("error_rate_vec").empty()) - throw std::runtime_error( - "sliding_window parameters: error_rate_vec must be non-empty"); - }; - decoding::config::register_decoder_schema(std::move(schema)); - } -}; -sliding_window_schema_registrar register_sliding_window_schema; -} // namespace - } // namespace cudaq::qec diff --git a/libs/qec/lib/realtime/CMakeLists.txt b/libs/qec/lib/realtime/CMakeLists.txt index d6bd20adc..1f0f55226 100644 --- a/libs/qec/lib/realtime/CMakeLists.txt +++ b/libs/qec/lib/realtime/CMakeLists.txt @@ -94,25 +94,6 @@ if(CMAKE_CUDA_COMPILER) message(STATUS "Found cuda-quantum realtime host dispatch library at ${CUDAQ_REALTIME_HOST_DISPATCH_LIBRARY}") endif() - # The device-graph dispatch API (cudaq_create_dispatch_graph_regular, - # cudaq_launch_dispatch_graph, cudaq_destroy_dispatch_graph) lives in - # libcudaq-realtime-dispatch, separate from libcudaq-realtime. It is - # relocatable device code, and qec_realtime_session resolves it at runtime - # via dlsym(RTLD_DEFAULT, ...) from the host executable's absorbed copy -- - # it must NOT be linked into cudaq-qec-realtime-decoding.so: a second - # device-linked copy of the dispatch kernel would live in a different CUDA - # module than the decoder's DEVICE_CALL handlers (captured from the - # executable's module), and cross-module device function pointer calls trap - # with cudaErrorIllegalInstruction. The find result is used by device-mode - # test executables, which absorb the archive themselves. - cudaq_qec_find_realtime_library(CUDAQ_REALTIME_DISPATCH_LIBRARY - NAMES cudaq-realtime-dispatch - PATH_SUFFIXES lib - ) - if(CUDAQ_REALTIME_DISPATCH_LIBRARY) - message(STATUS "Found cuda-quantum realtime dispatch library at ${CUDAQ_REALTIME_DISPATCH_LIBRARY}") - endif() - # CUDA-Q lowered `cudaq::device_call` host-dispatch integration needs the # DeviceCallService provider API from a realtime/device-call-enabled CUDA-Q # build. @@ -144,6 +125,7 @@ if(CMAKE_CUDA_COMPILER) if(CUDAQ_DEVICE_CALL_INCLUDE_DIR AND CUDAQ_DEVICE_CALL_RUNTIME_LIBRARY) set(CUDAQ_QEC_DEVICE_CALL_SERVICE_AVAILABLE TRUE) endif() + # Create static device library for device code add_library(cudaq-qec-realtime-cudevice STATIC @@ -180,7 +162,6 @@ if(CMAKE_CUDA_COMPILER) COMPONENT qec-lib ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ) - else() message(WARNING "cuda-quantum realtime headers not found. " "Device library cudaq-qec-realtime-device will not be built. " @@ -188,22 +169,7 @@ if(CMAKE_CUDA_COMPILER) endif() endif() -set(CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE "" CACHE FILEPATH - "Path to prebuilt proprietary realtime cudevice archive") -if(CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE) - if(NOT EXISTS "${CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE}") - message(FATAL_ERROR - "CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE does not exist: " - "${CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE}") - endif() - add_library(cudaq-qec-realtime-cudevice-proprietary STATIC IMPORTED GLOBAL) - set_target_properties(cudaq-qec-realtime-cudevice-proprietary PROPERTIES - IMPORTED_LOCATION "${CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE}") -endif() - -# Shared library for host-side API. The cqr host-dispatch service (server) and -# the simulated-QPU client wrappers live in their own sibling libraries built -# from ./decoding-server-cqr and ./simulation-cqr (added below), not here. +# Shared library for host-side API add_library(cudaq-qec-realtime-decoding SHARED realtime_decoding.cpp config.cpp @@ -230,7 +196,7 @@ if(CUDAQ_REALTIME_INCLUDE_DIR) CUDAQ_REALTIME_ROOT) endif() -target_link_options(cudaq-qec-realtime-decoding PRIVATE +target_link_options(cudaq-qec-realtime-decoding PUBLIC $<$:-Wl,--exclude-libs,ALL> ) @@ -263,24 +229,8 @@ install(TARGETS cudaq-qec-realtime-decoding # --------------------------------------------------------------------------- # RealtimePipeline shared library # Requires pre-installed cudaq-realtime (set CUDAQ_REALTIME_ROOT) -# -# EXPERIMENTAL / temporarily disabled: realtime_pipeline.cu relies on host -# dispatch API surface (skip_stream_sweep, caller-owned worker streams, and -# per-worker pre/post-launch hooks) that was removed upstream in CUDA-Q PR4770 -# ("Split graph launch dispatch from the host ring loop"). Until it is ported -# to the new graph_launch_engine API, keep this target off by default so CI -# builds green. Dependent unit tests (test_realtime_pipeline, -# test_realtime_predecoder_w_pymatching, hololink bridge) auto-skip when this -# target does not exist. # --------------------------------------------------------------------------- -option(CUDAQX_QEC_ENABLE_REALTIME_PIPELINE - "Build the experimental cudaq-realtime-pipeline library (needs porting to \ -the post-PR4770 CUDA-Q graph_launch_engine API)" OFF) - -if(NOT CUDAQX_QEC_ENABLE_REALTIME_PIPELINE) - message(STATUS "RealtimePipeline: skipping (experimental; disabled by " - "default -- set CUDAQX_QEC_ENABLE_REALTIME_PIPELINE=ON to build)") -elseif(CMAKE_CUDA_COMPILER AND CUDAQ_REALTIME_INCLUDE_DIR) +if(CMAKE_CUDA_COMPILER AND CUDAQ_REALTIME_INCLUDE_DIR) cudaq_qec_find_realtime_library(_CUDAQ_RT_LIB NAMES cudaq-realtime PATH_SUFFIXES lib @@ -342,11 +292,11 @@ else() endif() endif() -add_subdirectory(decoding-server-cqr) add_subdirectory(quantinuum) add_subdirectory(simulation) if(CUDAQ_QEC_DEVICE_CALL_SERVICE_AVAILABLE) - # cudaq-realtime (cqr) host-dispatch: simulated-QPU client that drives the - # decoding server over cudaq-realtime. + # cudaq-realtime (cqr) host-dispatch: the decoding-server skeleton and the + # simulated-QPU client that drives it over cudaq-realtime. + add_subdirectory(decoding-server-cqr) add_subdirectory(simulation-cqr) endif() diff --git a/libs/qec/lib/realtime/config.cpp b/libs/qec/lib/realtime/config.cpp index 389e82da0..32e75a3e3 100644 --- a/libs/qec/lib/realtime/config.cpp +++ b/libs/qec/lib/realtime/config.cpp @@ -7,268 +7,694 @@ ******************************************************************************/ #include "llvm/ADT/StringRef.h" -#include "llvm/Support/JSON.h" +#include "llvm/Support/Base64.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" #include "realtime_decoding.h" #include "cudaq/qec/decoder_config_payload.h" -#include "cudaq/qec/decoder_config_schema.h" #include "cudaq/qec/logger.h" #include "cudaq/qec/realtime/decoding_config.h" +#include #include #include -#include -#include -#include +#include + +// Helper function(s) to remove the optional wrapper from a type. +// Primary template for non-optional types +template +struct remove_optional { + using type = T; +}; +// Partial specialization for std::optional +template +struct remove_optional> { + using type = T; +}; +// Alias +template +using remove_optional_t = typename remove_optional::type; namespace cudaq::qec::decoding::config { -bool decoder_custom_args_t::operator==( - const decoder_custom_args_t &other) const { - return custom_args_maps_equal(map_, other.map_); +#define INSERT_ARG(arg_name) \ + do { \ + if (this->arg_name.has_value()) { \ + config_map.insert(#arg_name, this->arg_name.value()); \ + } \ + } while (false) + +#define INSERT_ARG_PLAIN(arg_name) \ + do { \ + config_map.insert(#arg_name, this->arg_name); \ + } while (false) + +#define GET_ARG(arg_name) \ + do { \ + if (map.contains(#arg_name)) { \ + config.arg_name = \ + map.get>(#arg_name); \ + } \ + } while (false) + +#define GET_ARG_PLAIN(arg_name) \ + do { \ + if (map.contains(#arg_name)) { \ + config.arg_name = \ + map.get>(#arg_name); \ + } \ + } while (false) + +// ------ relay_bp_config ------ +cudaqx::heterogeneous_map srelay_bp_config::to_heterogeneous_map() const { + cudaqx::heterogeneous_map config_map; + + INSERT_ARG(pre_iter); + INSERT_ARG(num_sets); + INSERT_ARG(stopping_criterion); + INSERT_ARG(stop_nconv); + + return config_map; } -void decoder_config::validate_custom_args() const { - config::validate_custom_args(type, decoder_custom_args.map()); +srelay_bp_config +srelay_bp_config::from_heterogeneous_map(const cudaqx::heterogeneous_map &map) { + srelay_bp_config config; + GET_ARG(pre_iter); + GET_ARG(num_sets); + GET_ARG(stopping_criterion); + GET_ARG(stop_nconv); + + return config; } +// ------ nv_qldpc_decoder_config ------ cudaqx::heterogeneous_map -decoder_config::decoder_custom_args_to_heterogeneous_map() const { - auto args = decoder_custom_args.map(); - if (const auto *schema = find_decoder_schema(type)) { - // Same normalization on every consumer path: non-schema keys are - // warned-and-dropped (they could never round-trip through YAML, so a - // local decoder must not see them either) and schema-declared defaults - // are materialized. YAML emission serializes this same map, so a config - // reaches a local decoder and a remote target identically. - drop_non_schema_keys(*schema, args); - materialize_default_args(*schema, args); +nv_qldpc_decoder_config::to_heterogeneous_map() const { + cudaqx::heterogeneous_map config_map; + + INSERT_ARG(use_sparsity); + INSERT_ARG(error_rate); + INSERT_ARG(error_rate_vec); + INSERT_ARG(max_iterations); + INSERT_ARG(n_threads); + INSERT_ARG(use_osd); + INSERT_ARG(osd_method); + INSERT_ARG(osd_order); + INSERT_ARG(bp_batch_size); + INSERT_ARG(osd_batch_size); + INSERT_ARG(iter_per_check); + INSERT_ARG(clip_value); + INSERT_ARG(bp_method); + INSERT_ARG(scale_factor); + INSERT_ARG(proc_float); + INSERT_ARG(gamma0); + INSERT_ARG(gamma_dist); + INSERT_ARG(explicit_gammas); + INSERT_ARG(bp_seed); + INSERT_ARG(composition); + // srelay_config needs to be converted to heterogeneous_map for decoder + // compatibility + if (srelay_config.has_value()) { + config_map.insert("srelay_config", srelay_config->to_heterogeneous_map()); } - return args; + + return config_map; } -void multi_decoder_config::validate_custom_args() const { - for (const auto &decoder : decoders) - decoder.validate_custom_args(); +nv_qldpc_decoder_config nv_qldpc_decoder_config::from_heterogeneous_map( + const cudaqx::heterogeneous_map &map) { + nv_qldpc_decoder_config config; + GET_ARG(use_sparsity); + GET_ARG(error_rate); + GET_ARG(error_rate_vec); + GET_ARG(max_iterations); + GET_ARG(n_threads); + GET_ARG(use_osd); + GET_ARG(osd_method); + GET_ARG(osd_order); + GET_ARG(bp_batch_size); + GET_ARG(osd_batch_size); + GET_ARG(iter_per_check); + GET_ARG(clip_value); + GET_ARG(bp_method); + GET_ARG(scale_factor); + GET_ARG(proc_float); + GET_ARG(gamma0); + GET_ARG(gamma_dist); + GET_ARG(explicit_gammas); + GET_ARG(bp_seed); + GET_ARG(composition); + // Special handling for srelay_config - it might be stored as a nested + // heterogeneous_map + if (map.contains("srelay_config")) { + try { + // Try to get it as a srelay_bp_config first (for C++ -> C++) + config.srelay_config = map.get("srelay_config"); + } catch (...) { + // If that fails, try to get it as a heterogeneous_map (for Python + // round-trip) + auto nested_map = map.get("srelay_config"); + config.srelay_config = + srelay_bp_config::from_heterogeneous_map(nested_map); + } + } + + return config; } -// Post-parse pass over a schema-parsed custom-args map: materialize defaulted -// discriminated sections, then run the canonical registry validation -// (required keys, per-schema hooks; its unknown-key check is a no-op here -// because the parser already rejected unknown keys). Only invoked when the -// section was present in the input document, mirroring the previous behavior -// where an absent decoder_custom_args section skipped its mapping (and -// therefore its required-key checks) entirely. -static void finalize_parsed_args(const decoder_schema &schema, - cudaqx::heterogeneous_map &map, - const std::string &context) { - materialize_default_args(schema, map); - validate_custom_args(schema, map, context); +// ------ multi_error_lut_config ------ +cudaqx::heterogeneous_map multi_error_lut_config::to_heterogeneous_map() const { + cudaqx::heterogeneous_map config_map; + INSERT_ARG(lut_error_depth); + return config_map; } -} // namespace cudaq::qec::decoding::config +multi_error_lut_config multi_error_lut_config::from_heterogeneous_map( + const cudaqx::heterogeneous_map &map) { + multi_error_lut_config config; + GET_ARG(lut_error_depth); + return config; +} -LLVM_YAML_IS_SEQUENCE_VECTOR(std::vector) -LLVM_YAML_IS_SEQUENCE_VECTOR(cudaq::qec::decoding::config::decoder_config) +// ------ single_error_lut_config ------ +cudaqx::heterogeneous_map +single_error_lut_config::to_heterogeneous_map() const { + cudaqx::heterogeneous_map config_map; + return config_map; +} -namespace llvm::yaml { +single_error_lut_config single_error_lut_config::from_heterogeneous_map( + const cudaqx::heterogeneous_map &map) { + single_error_lut_config config; + return config; +} -// Binds a heterogeneous_map to the decoder_schema that describes it so the -// generic mapping traits below can convert between the two. The schema drives -// everything: which keys are legal, the canonical storage type of each value, -// and how nested sections resolve their schemas. -struct schema_binding { - cudaqx::heterogeneous_map *map = nullptr; - const cudaq::qec::decoding::config::decoder_schema *schema = nullptr; -}; +cudaqx::heterogeneous_map global_decoder_config_to_heterogeneous_map( + const global_decoder_config &global_decoder_params) { + return std::visit( + [](const auto ¶ms) -> cudaqx::heterogeneous_map { + using config_t = std::decay_t; + if constexpr (std::is_same_v) { + return cudaqx::heterogeneous_map(); + } else { + return params.to_heterogeneous_map(); + } + }, + global_decoder_params); +} -namespace { +global_decoder_config global_decoder_config_from_heterogeneous_map( + const cudaqx::heterogeneous_map &map, + const std::optional &global_decoder) { + if (!global_decoder.has_value()) { + throw std::runtime_error( + "global_decoder_params present but global_decoder is not set."); + } -template -void input_schema_scalar(IO &io, const std::string &key, - cudaqx::heterogeneous_map &map) { - T value{}; - io.mapRequired(key.c_str(), value); - map.insert(key, value); + if (global_decoder.value() == "pymatching") { + return pymatching_config::from_heterogeneous_map(map); + } + + if (global_decoder.value() == "chromobius") { + return chromobius_config::from_heterogeneous_map(map); + } + + throw std::runtime_error( + "global_decoder_params does not support global_decoder '" + + global_decoder.value() + "'."); } -template -void output_schema_scalar(IO &io, const std::string &key, - const cudaqx::heterogeneous_map &map) { - T value = map.get(key); - io.mapRequired(key.c_str(), value); +global_decoder_config default_global_decoder_params( + const std::optional &global_decoder) { + if (!global_decoder.has_value()) + return std::monostate{}; + + if (global_decoder.value() == "pymatching") + return pymatching_config{}; + + if (global_decoder.value() == "chromobius") + return chromobius_config{}; + + return std::monostate{}; } -} // namespace +void validate_global_decoder_params( + const global_decoder_config &global_decoder_params, + const std::optional &global_decoder); -template <> -struct CustomMappingTraits { - using param_kind = cudaq::qec::decoding::config::param_kind; - using param_spec = cudaq::qec::decoding::config::param_spec; - - static void inputOne(IO &io, StringRef key, schema_binding &binding) { - const std::string key_str = key.str(); - const param_spec *spec = nullptr; - for (const auto &candidate : binding.schema->params) { - if (candidate.key == key_str) { - spec = &candidate; +global_decoder_config global_decoder_config_from_value( + const std::any &val, const std::optional &global_decoder) { + if (!global_decoder.has_value()) { + throw std::runtime_error( + "global_decoder_params present but global_decoder is not set."); + } + + if (auto *global_cfg = std::any_cast(&val)) { + validate_global_decoder_params(*global_cfg, global_decoder); + return *global_cfg; + } + + if (auto *nested_map = std::any_cast(&val)) { + return global_decoder_config_from_heterogeneous_map(*nested_map, + global_decoder); + } + + global_decoder_config parsed_params; + if (auto *pymatching_cfg = std::any_cast(&val)) { + parsed_params = *pymatching_cfg; + } else if (auto *chromobius_cfg = std::any_cast(&val)) { + parsed_params = *chromobius_cfg; + } else { + throw std::runtime_error( + "global_decoder_params has an unsupported value type for " + "global_decoder '" + + global_decoder.value() + "'."); + } + + validate_global_decoder_params(parsed_params, global_decoder); + return parsed_params; +} + +void validate_global_decoder_params( + const global_decoder_config &global_decoder_params, + const std::optional &global_decoder) { + if (std::holds_alternative(global_decoder_params)) + return; + + if (!global_decoder.has_value()) { + throw std::runtime_error( + "global_decoder_params present but global_decoder is not set."); + } + + if (global_decoder.value() == "pymatching" && + std::holds_alternative(global_decoder_params)) { + return; + } + + if (global_decoder.value() == "chromobius" && + std::holds_alternative(global_decoder_params)) { + return; + } + + throw std::runtime_error( + "global_decoder_params type does not match global_decoder '" + + global_decoder.value() + "'."); +} + +// ------ pymatching_config ------ +cudaqx::heterogeneous_map pymatching_config::to_heterogeneous_map() const { + cudaqx::heterogeneous_map config_map; + + INSERT_ARG(error_rate_vec); + INSERT_ARG(merge_strategy); + + return config_map; +} + +pymatching_config pymatching_config::from_heterogeneous_map( + const cudaqx::heterogeneous_map &map) { + pymatching_config config; + GET_ARG(error_rate_vec); + GET_ARG(merge_strategy); + return config; +} + +// ------ chromobius_config ------ +cudaqx::heterogeneous_map chromobius_config::to_heterogeneous_map() const { + cudaqx::heterogeneous_map config_map; + + INSERT_ARG(drop_mobius_errors_involving_remnant_errors); + INSERT_ARG(ignore_decomposition_failures); + INSERT_ARG(include_coords_in_mobius_dem); + INSERT_ARG(return_weight); + INSERT_ARG(write_mobius_match_to_stderr); + + return config_map; +} + +chromobius_config chromobius_config::from_heterogeneous_map( + const cudaqx::heterogeneous_map &map) { + chromobius_config config; + GET_ARG(drop_mobius_errors_involving_remnant_errors); + GET_ARG(ignore_decomposition_failures); + GET_ARG(include_coords_in_mobius_dem); + GET_ARG(return_weight); + GET_ARG(write_mobius_match_to_stderr); + return config; +} + +// ------ trt_decoder_config ------ +cudaqx::heterogeneous_map trt_decoder_config::to_heterogeneous_map() const { + cudaqx::heterogeneous_map config_map; + + INSERT_ARG(onnx_load_path); + INSERT_ARG(engine_load_path); + INSERT_ARG(engine_save_path); + INSERT_ARG(precision); + INSERT_ARG(memory_workspace); + INSERT_ARG(batch_size); + INSERT_ARG(use_cuda_graph); + INSERT_ARG(global_decoder); + auto effective_global_decoder_params = global_decoder_params; + if (std::holds_alternative(effective_global_decoder_params)) + effective_global_decoder_params = + default_global_decoder_params(global_decoder); + if (!std::holds_alternative( + effective_global_decoder_params)) { + validate_global_decoder_params(effective_global_decoder_params, + global_decoder); + config_map.insert("global_decoder_params", + global_decoder_config_to_heterogeneous_map( + effective_global_decoder_params)); + } + + return config_map; +} + +trt_decoder_config trt_decoder_config::from_heterogeneous_map( + const cudaqx::heterogeneous_map &map) { + trt_decoder_config config; + GET_ARG(onnx_load_path); + GET_ARG(engine_load_path); + GET_ARG(engine_save_path); + GET_ARG(precision); + GET_ARG(memory_workspace); + GET_ARG(batch_size); + GET_ARG(use_cuda_graph); + GET_ARG(global_decoder); + if (map.contains("global_decoder_params")) { + for (const auto &[key, val] : map) { + if (key == "global_decoder_params") { + config.global_decoder_params = + global_decoder_config_from_value(val, config.global_decoder); break; } } - if (!spec) - throw std::runtime_error("Unknown key '" + key_str + "' in '" + - binding.schema->name + "' parameters."); - - switch (spec->kind) { - case param_kind::boolean: - input_schema_scalar(io, key_str, *binding.map); - break; - case param_kind::int32: - input_schema_scalar(io, key_str, *binding.map); - break; - case param_kind::uint64: - input_schema_scalar(io, key_str, *binding.map); - break; - case param_kind::f64: - input_schema_scalar(io, key_str, *binding.map); - break; - case param_kind::string: - input_schema_scalar(io, key_str, *binding.map); - break; - case param_kind::f64_vec: - input_schema_scalar>(io, key_str, *binding.map); - break; - case param_kind::f64_matrix: - input_schema_scalar>>(io, key_str, - *binding.map); - break; - case param_kind::subschema: { - const auto *nested_schema = - cudaq::qec::decoding::config::find_decoder_schema(spec->subschema); - if (!nested_schema) - throw std::runtime_error("No schema registered under '" + - spec->subschema + "' (needed to parse '" + - key_str + "')."); - cudaqx::heterogeneous_map nested; - schema_binding nested_binding{&nested, nested_schema}; - io.mapRequired(key_str.c_str(), nested_binding); - binding.map->insert(key_str, nested); - break; - } - case param_kind::discriminated: { - // The nested schema is named by a sibling key. Read it through the IO - // (document order does not matter; mapping keys are random access). - std::string discriminator_value; - io.mapOptional(spec->discriminator.c_str(), discriminator_value); - if (discriminator_value.empty()) - throw std::runtime_error("'" + key_str + "' is present but '" + - spec->discriminator + "' is not set."); - const auto *nested_schema = - cudaq::qec::decoding::config::find_decoder_schema( - discriminator_value); - if (!nested_schema) - throw std::runtime_error( - "'" + key_str + "' does not support " + spec->discriminator + " '" + - discriminator_value + - "': no parameter schema is registered under that name."); - cudaqx::heterogeneous_map nested; - schema_binding nested_binding{&nested, nested_schema}; - io.mapRequired(key_str.c_str(), nested_binding); - binding.map->insert(key_str, nested); - break; - } + } else { + config.global_decoder_params = + default_global_decoder_params(config.global_decoder); + } + + return config; +} + +// ------ sliding_window_config ------ +cudaqx::heterogeneous_map sliding_window_config::to_heterogeneous_map() const { + cudaqx::heterogeneous_map config_map; + INSERT_ARG(window_size); + INSERT_ARG(step_size); + INSERT_ARG(num_syndromes_per_round); + INSERT_ARG(num_boundary_syndromes); + INSERT_ARG(straddle_start_round); + INSERT_ARG(straddle_end_round); + INSERT_ARG_PLAIN(error_rate_vec); + INSERT_ARG_PLAIN(inner_decoder_name); + + // Handle concrete inner decoder configs + cudaqx::heterogeneous_map inner_decoder_params; + if (single_error_lut_params.has_value()) { + inner_decoder_params = + single_error_lut_params.value().to_heterogeneous_map(); + } else if (multi_error_lut_params.has_value()) { + inner_decoder_params = + multi_error_lut_params.value().to_heterogeneous_map(); + } else if (nv_qldpc_decoder_params.has_value()) { + inner_decoder_params = + nv_qldpc_decoder_params.value().to_heterogeneous_map(); + } + if (!inner_decoder_params.empty()) { + config_map.insert("inner_decoder_params", inner_decoder_params); + } + + return config_map; +} + +sliding_window_config sliding_window_config::from_heterogeneous_map( + const cudaqx::heterogeneous_map &map) { + sliding_window_config config; + GET_ARG(window_size); + GET_ARG(step_size); + GET_ARG(num_syndromes_per_round); + GET_ARG(num_boundary_syndromes); + GET_ARG(straddle_start_round); + GET_ARG(straddle_end_round); + GET_ARG_PLAIN(error_rate_vec); + GET_ARG_PLAIN(inner_decoder_name); + + // Reconstruct inner decoder configs based on the decoder type + if (!config.inner_decoder_name.empty() && + map.contains("inner_decoder_params")) { + const auto &inner_decoder_params = + map.get("inner_decoder_params"); + const std::string &decoder_name = config.inner_decoder_name; + + if (decoder_name == "single_error_lut") { + config.single_error_lut_params = + single_error_lut_config::from_heterogeneous_map(inner_decoder_params); + } else if (decoder_name == "multi_error_lut") { + config.multi_error_lut_params = + multi_error_lut_config::from_heterogeneous_map(inner_decoder_params); + } else if (decoder_name == "nv-qldpc-decoder") { + config.nv_qldpc_decoder_params = + nv_qldpc_decoder_config::from_heterogeneous_map(inner_decoder_params); } } - static void output(IO &io, schema_binding &binding) { - // Only schema keys are emitted; surface anything else (a typo in a - // programmatically built map) instead of dropping it silently. - for (const auto &kv : *binding.map) { - bool known = false; - for (const auto &spec : binding.schema->params) { - if (spec.key == kv.first) { - known = true; - break; - } + return config; +} + +#undef INSERT_ARG +#undef GET_ARG + +} // namespace cudaq::qec::decoding::config + +LLVM_YAML_IS_SEQUENCE_VECTOR(std::vector) +LLVM_YAML_IS_SEQUENCE_VECTOR(cudaq::qec::decoding::config::decoder_config) + +namespace llvm::yaml { + +template <> +struct MappingTraits { + static void mapping(IO &io, + cudaq::qec::decoding::config::srelay_bp_config &config) { + io.mapOptional("pre_iter", config.pre_iter); + io.mapOptional("num_sets", config.num_sets); + io.mapOptional("stopping_criterion", config.stopping_criterion); + io.mapOptional("stop_nconv", config.stop_nconv); + } +}; + +template <> +struct MappingTraits { + static void + mapping(IO &io, + cudaq::qec::decoding::config::nv_qldpc_decoder_config &config) { + io.mapOptional("use_sparsity", config.use_sparsity); + io.mapOptional("error_rate", config.error_rate); + io.mapOptional("error_rate_vec", config.error_rate_vec); + io.mapOptional("max_iterations", config.max_iterations); + io.mapOptional("n_threads", config.n_threads); + io.mapOptional("use_osd", config.use_osd); + io.mapOptional("osd_method", config.osd_method); + io.mapOptional("osd_order", config.osd_order); + io.mapOptional("bp_batch_size", config.bp_batch_size); + io.mapOptional("osd_batch_size", config.osd_batch_size); + io.mapOptional("iter_per_check", config.iter_per_check); + io.mapOptional("clip_value", config.clip_value); + io.mapOptional("bp_method", config.bp_method); + io.mapOptional("scale_factor", config.scale_factor); + io.mapOptional("proc_float", config.proc_float); + io.mapOptional("gamma0", config.gamma0); + io.mapOptional("gamma_dist", config.gamma_dist); + io.mapOptional("explicit_gammas", config.explicit_gammas); + io.mapOptional("bp_seed", config.bp_seed); + io.mapOptional("srelay_config", config.srelay_config); + io.mapOptional("composition", config.composition); + } +}; + +template <> +struct MappingTraits { + static void + mapping(IO &io, + cudaq::qec::decoding::config::multi_error_lut_config &config) { + io.mapOptional("lut_error_depth", config.lut_error_depth); + } +}; + +template <> +struct MappingTraits { + static void + mapping(IO &io, + cudaq::qec::decoding::config::single_error_lut_config &config) {} +}; + +template <> +struct MappingTraits { + static void + mapping(IO &io, cudaq::qec::decoding::config::global_decoder_config &config) { + using namespace cudaq::qec::decoding::config; + + if (io.outputting()) { + if (std::holds_alternative(config)) { + return; + } + + if (std::holds_alternative(config)) { + auto ¶ms = std::get(config); + io.mapOptional("merge_strategy", params.merge_strategy); + io.mapOptional("error_rate_vec", params.error_rate_vec); + return; } - if (!known) - CUDA_QEC_WARN("Key '{}' is not in the '{}' parameter schema; it is " - "omitted from the emitted YAML.", - kv.first, binding.schema->name); + + auto ¶ms = std::get(config); + io.mapOptional("drop_mobius_errors_involving_remnant_errors", + params.drop_mobius_errors_involving_remnant_errors); + io.mapOptional("ignore_decomposition_failures", + params.ignore_decomposition_failures); + io.mapOptional("include_coords_in_mobius_dem", + params.include_coords_in_mobius_dem); + io.mapOptional("return_weight", params.return_weight); + io.mapOptional("write_mobius_match_to_stderr", + params.write_mobius_match_to_stderr); + return; } - // Emit in schema declaration order so output is deterministic. - for (const auto &spec : binding.schema->params) { - if (!binding.map->contains(spec.key)) - continue; - switch (spec.kind) { - case param_kind::boolean: - output_schema_scalar(io, spec.key, *binding.map); - break; - case param_kind::int32: - output_schema_scalar(io, spec.key, *binding.map); - break; - case param_kind::uint64: - output_schema_scalar(io, spec.key, *binding.map); - break; - case param_kind::f64: - output_schema_scalar(io, spec.key, *binding.map); - break; - case param_kind::string: - output_schema_scalar(io, spec.key, *binding.map); - break; - case param_kind::f64_vec: - output_schema_scalar>(io, spec.key, *binding.map); - break; - case param_kind::f64_matrix: - output_schema_scalar>>(io, spec.key, - *binding.map); - break; - case param_kind::subschema: { - const auto *nested_schema = - cudaq::qec::decoding::config::find_decoder_schema(spec.subschema); - if (!nested_schema) - throw std::runtime_error("No schema registered under '" + - spec.subschema + "' (needed to emit '" + - spec.key + "')."); - auto nested = binding.map->get(spec.key); - schema_binding nested_binding{&nested, nested_schema}; - io.mapRequired(spec.key.c_str(), nested_binding); - break; + + // Input cannot be decoded safely here because the variant type depends on + // the parent trt_decoder_config.global_decoder value. The TRT mapping + // below dispatches with that context. + throw std::runtime_error( + "global_decoder_config YAML input requires trt_decoder_config " + "global_decoder context."); + } +}; + +template <> +struct MappingTraits { + static void mapping(IO &io, + cudaq::qec::decoding::config::pymatching_config &config) { + io.mapOptional("error_rate_vec", config.error_rate_vec); + io.mapOptional("merge_strategy", config.merge_strategy); + } +}; + +template <> +struct MappingTraits { + static void mapping(IO &io, + cudaq::qec::decoding::config::chromobius_config &config) { + io.mapOptional("drop_mobius_errors_involving_remnant_errors", + config.drop_mobius_errors_involving_remnant_errors); + io.mapOptional("ignore_decomposition_failures", + config.ignore_decomposition_failures); + io.mapOptional("include_coords_in_mobius_dem", + config.include_coords_in_mobius_dem); + io.mapOptional("return_weight", config.return_weight); + io.mapOptional("write_mobius_match_to_stderr", + config.write_mobius_match_to_stderr); + } +}; + +template <> +struct MappingTraits { + static void + mapping(IO &io, cudaq::qec::decoding::config::trt_decoder_config &config) { + io.mapOptional("onnx_load_path", config.onnx_load_path); + io.mapOptional("engine_load_path", config.engine_load_path); + io.mapOptional("engine_save_path", config.engine_save_path); + io.mapOptional("precision", config.precision); + io.mapOptional("memory_workspace", config.memory_workspace); + io.mapOptional("batch_size", config.batch_size); + io.mapOptional("use_cuda_graph", config.use_cuda_graph); + io.mapOptional("global_decoder", config.global_decoder); + + if (io.outputting()) { + auto global_decoder_params = config.global_decoder_params; + if (std::holds_alternative(global_decoder_params)) { + global_decoder_params = + cudaq::qec::decoding::config::default_global_decoder_params( + config.global_decoder); } - case param_kind::discriminated: { - std::string discriminator_value; - if (binding.map->contains(spec.discriminator)) - discriminator_value = - binding.map->get(spec.discriminator); - const auto *nested_schema = - discriminator_value.empty() - ? nullptr - : cudaq::qec::decoding::config::find_decoder_schema( - discriminator_value); - if (!nested_schema) - throw std::runtime_error("'" + spec.key + - "' is present but no parameter schema is " - "registered for " + - spec.discriminator + " '" + - discriminator_value + "'."); - auto nested = binding.map->get(spec.key); - schema_binding nested_binding{&nested, nested_schema}; - io.mapRequired(spec.key.c_str(), nested_binding); - break; + if (std::holds_alternative(global_decoder_params)) { + return; } + + cudaq::qec::decoding::config::validate_global_decoder_params( + global_decoder_params, config.global_decoder); + if (config.global_decoder.value() == "pymatching") { + io.mapOptional( + "global_decoder_params", + std::get( + global_decoder_params)); + } else if (config.global_decoder.value() == "chromobius") { + io.mapOptional( + "global_decoder_params", + std::get( + global_decoder_params)); + } + return; + } + + if (config.global_decoder.has_value() && + config.global_decoder.value() == "pymatching") { + std::optional params; + io.mapOptional("global_decoder_params", params); + if (params.has_value()) + config.global_decoder_params = std::move(params.value()); + else + config.global_decoder_params = + cudaq::qec::decoding::config::default_global_decoder_params( + config.global_decoder); + } else if (config.global_decoder.has_value() && + config.global_decoder.value() == "chromobius") { + std::optional params; + io.mapOptional("global_decoder_params", params); + if (params.has_value()) + config.global_decoder_params = std::move(params.value()); + else + config.global_decoder_params = + cudaq::qec::decoding::config::default_global_decoder_params( + config.global_decoder); + } else { + // Use a throwaway value only to detect whether the key was present. Do + // not assign it to config.global_decoder_params: without a supported + // global_decoder name, there is no safe variant type to parse into. + std::optional params; + io.mapOptional("global_decoder_params", params); + if (params.has_value()) { + if (config.global_decoder.has_value()) { + throw std::runtime_error( + "global_decoder_params does not support global_decoder '" + + config.global_decoder.value() + "'."); + } else { + throw std::runtime_error( + "global_decoder_params present but global_decoder is not set."); + } } } } }; template <> -struct ScalarEnumerationTraits { +struct MappingTraits { static void - enumeration(IO &io, cudaq::qec::decoding::config::DecoderTransport &value) { - io.enumCase(value, "cpu_roce", - cudaq::qec::decoding::config::DecoderTransport::cpu_roce); - io.enumCase(value, "gpu_roce", - cudaq::qec::decoding::config::DecoderTransport::gpu_roce); + mapping(IO &io, cudaq::qec::decoding::config::sliding_window_config &config) { + io.mapOptional("window_size", config.window_size); + io.mapOptional("step_size", config.step_size); + io.mapOptional("num_syndromes_per_round", config.num_syndromes_per_round); + io.mapOptional("num_boundary_syndromes", config.num_boundary_syndromes); + io.mapOptional("straddle_start_round", config.straddle_start_round); + io.mapOptional("straddle_end_round", config.straddle_end_round); + io.mapRequired("error_rate_vec", config.error_rate_vec); + io.mapRequired("inner_decoder_name", config.inner_decoder_name); + + // Concrete inner decoder configurations + if (config.inner_decoder_name == "single_error_lut") { + io.mapOptional("inner_decoder_params", config.single_error_lut_params); + } else if (config.inner_decoder_name == "multi_error_lut") { + io.mapOptional("inner_decoder_params", config.multi_error_lut_params); + } else if (config.inner_decoder_name == "nv-qldpc-decoder") { + io.mapOptional("inner_decoder_params", config.nv_qldpc_decoder_params); + } } }; @@ -278,8 +704,6 @@ struct MappingTraits { cudaq::qec::decoding::config::decoder_config &config) { io.mapRequired("id", config.id); io.mapRequired("type", config.type); - io.mapOptional("transport", config.transport, - cudaq::qec::decoding::config::DecoderTransport::cpu_roce); io.mapOptional("cuda_device_id", config.cuda_device_id); io.mapRequired("block_size", config.block_size); io.mapRequired("syndrome_size", config.syndrome_size); @@ -335,52 +759,33 @@ struct MappingTraits { } } } - - // Convert decoder_custom_args through the schema registered for this - // decoder type. When no schema is registered, the key is intentionally - // left unconsumed on input so the YAML parser's strict unknown-key check - // rejects the section -- a decoder must register a schema (from its own - // plugin library) to accept custom args. - const auto *schema = - cudaq::qec::decoding::config::find_decoder_schema(config.type); - if (io.outputting()) { - if (!config.decoder_custom_args.empty()) { - if (!schema) { - // Match the historical emission behavior (args for unknown types - // were silently dropped) so configuration flows still fail with a - // status code at decoder construction rather than throwing here. - CUDA_QEC_WARN( - "decoder_custom_args set for decoder type '{}' but no parameter " - "schema is registered under that name; the args are omitted " - "from the emitted YAML.", - config.type); - } else { - // Emit the same normalized map the constructor-facing path - // produces (non-schema keys dropped, defaults materialized), so a - // programmatically built config serializes identically on first - // emission and after a YAML round trip -- e.g. a trt config with - // only `global_decoder` set gains `global_decoder_params: {}` - // here, not just after re-parsing. - auto args_map = config.decoder_custom_args_to_heterogeneous_map(); - schema_binding binding{&args_map, schema}; - io.mapRequired("decoder_custom_args", binding); - } - } - } else if (schema) { - bool args_present = false; - for (const auto key : io.keys()) { - if (key == "decoder_custom_args") { - args_present = true; - break; - } - } - cudaqx::heterogeneous_map args_map; - schema_binding binding{&args_map, schema}; - io.mapOptional("decoder_custom_args", binding); - if (args_present) - cudaq::qec::decoding::config::finalize_parsed_args( - *schema, args_map, "decoder_custom_args (" + config.type + ")"); - config.decoder_custom_args = args_map; +#define INIT_AND_MAP_DECODER_CUSTOM_ARGS(type) \ + do { \ + if (!std::holds_alternative(config.decoder_custom_args)) { \ + config.decoder_custom_args = type(); \ + } \ + io.mapOptional("decoder_custom_args", \ + std::get(config.decoder_custom_args)); \ + } while (false) + + if (config.type == "nv-qldpc-decoder") { + INIT_AND_MAP_DECODER_CUSTOM_ARGS( + cudaq::qec::decoding::config::nv_qldpc_decoder_config); + } else if (config.type == "multi_error_lut") { + INIT_AND_MAP_DECODER_CUSTOM_ARGS( + cudaq::qec::decoding::config::multi_error_lut_config); + } else if (config.type == "single_error_lut") { + INIT_AND_MAP_DECODER_CUSTOM_ARGS( + cudaq::qec::decoding::config::single_error_lut_config); + } else if (config.type == "trt_decoder") { + INIT_AND_MAP_DECODER_CUSTOM_ARGS( + cudaq::qec::decoding::config::trt_decoder_config); + } else if (config.type == "sliding_window") { + INIT_AND_MAP_DECODER_CUSTOM_ARGS( + cudaq::qec::decoding::config::sliding_window_config); + } else if (config.type == "pymatching") { + INIT_AND_MAP_DECODER_CUSTOM_ARGS( + cudaq::qec::decoding::config::pymatching_config); } } }; @@ -403,9 +808,6 @@ cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( multi_decoder_config config; llvm::yaml::Input yaml_in(yaml_str); yaml_in >> config; - if (const auto error = yaml_in.error()) - throw std::runtime_error("Invalid decoder configuration YAML: " + - error.message()); return config; } @@ -424,9 +826,6 @@ cudaq::qec::decoding::config::decoder_config::from_yaml_str( decoder_config config; llvm::yaml::Input yaml_in(yaml_str); yaml_in >> config; - if (const auto error = yaml_in.error()) - throw std::runtime_error("Invalid decoder configuration YAML: " + - error.message()); return config; } @@ -441,251 +840,8 @@ cudaq::qec::decoding::config::decoder_config::to_yaml_str(int column_wrap) { namespace cudaq::qec::decoding::config { -// --------------------------------------------------------------------------- -// JSON Schema export -// -// Translates the registered decoder parameter schemas plus the fixed -// decoder_config envelope (the fields MappingTraits maps -// above) into a JSON Schema draft 2020-12 document, so standard tooling can -// validate user-provided configuration YAML offline. The document is a -// snapshot of what this installation can parse: it enumerates the schemas -// registered at call time, exactly as the runtime parser resolves them. -// --------------------------------------------------------------------------- - -namespace { - -// JSON-pointer token escaping for schema names used inside $ref paths. -std::string json_pointer_escape(const std::string &name) { - std::string out; - for (char c : name) { - if (c == '~') - out += "~0"; - else if (c == '/') - out += "~1"; - else - out += c; - } - return out; -} - -std::string params_ref(const std::string &name) { - return "#/$defs/decoder_params/" + json_pointer_escape(name); -} - -llvm::json::Object json_schema_for_param(const param_spec &spec) { - using k = param_kind; - switch (spec.kind) { - case k::boolean: - return llvm::json::Object{{"type", "boolean"}}; - case k::int32: - return llvm::json::Object{{"type", "integer"}}; - case k::uint64: - return llvm::json::Object{{"type", "integer"}, {"minimum", 0}}; - case k::f64: - return llvm::json::Object{{"type", "number"}}; - case k::string: - return llvm::json::Object{{"type", "string"}}; - case k::f64_vec: - return llvm::json::Object{ - {"type", "array"}, {"items", llvm::json::Object{{"type", "number"}}}}; - case k::f64_matrix: - return llvm::json::Object{ - {"type", "array"}, - {"items", llvm::json::Object{ - {"type", "array"}, - {"items", llvm::json::Object{{"type", "number"}}}}}}; - case k::subschema: - return llvm::json::Object{{"$ref", params_ref(spec.subschema)}}; - case k::discriminated: - // The concrete shape is selected by the discriminator value; the - // dispatch clauses emitted below refine this. - return llvm::json::Object{{"type", "object"}}; - } - return llvm::json::Object{}; -} - -llvm::json::Array registered_name_array(const std::vector &names) { - llvm::json::Array arr; - for (const auto &name : names) - arr.push_back(name); - return arr; -} - -llvm::json::Object -decoder_params_json_schema(const decoder_schema &schema, - const std::vector &all_names) { - llvm::json::Object properties; - llvm::json::Array required; - llvm::json::Array all_of; - for (const auto &spec : schema.params) { - properties[spec.key] = json_schema_for_param(spec); - if (spec.required) - required.push_back(spec.key); - if (spec.kind == param_kind::discriminated) { - // When the section is present, its discriminator must be present and - // name a registered schema (mirrors the parser's checks). - all_of.push_back(llvm::json::Object{ - {"if", llvm::json::Object{{"required", llvm::json::Array{spec.key}}}}, - {"then", - llvm::json::Object{ - {"required", llvm::json::Array{spec.discriminator}}, - {"properties", - llvm::json::Object{ - {spec.discriminator, - llvm::json::Object{ - {"enum", registered_name_array(all_names)}}}}}}}}); - // Each candidate discriminator value selects that schema for the - // section. - for (const auto &name : all_names) - all_of.push_back(llvm::json::Object{ - {"if", - llvm::json::Object{ - {"properties", - llvm::json::Object{{spec.discriminator, - llvm::json::Object{{"const", name}}}}}, - {"required", - llvm::json::Array{spec.discriminator, spec.key}}}}, - {"then", llvm::json::Object{ - {"properties", - llvm::json::Object{ - {spec.key, llvm::json::Object{ - {"$ref", params_ref(name)}}}}}}}}); - } - } - llvm::json::Object out{{"type", "object"}, - {"additionalProperties", false}, - {"properties", std::move(properties)}}; - if (!required.empty()) - out["required"] = std::move(required); - if (!all_of.empty()) - out["allOf"] = std::move(all_of); - return out; -} - -} // namespace - -std::string decoder_config_json_schema() { - const auto names = registered_decoder_schema_names(); - - llvm::json::Object decoder_params; - for (const auto &name : names) - decoder_params[name] = - decoder_params_json_schema(*find_decoder_schema(name), names); - - // The fixed decoder_config envelope; keep in sync with - // MappingTraits above. - llvm::json::Object config_properties{ - {"id", llvm::json::Object{{"type", "integer"}}}, - {"type", llvm::json::Object{{"type", "string"}}}, - {"transport", - llvm::json::Object{{"enum", llvm::json::Array{"cpu_roce", "gpu_roce"}}}}, - {"cuda_device_id", - llvm::json::Object{{"type", "integer"}, {"minimum", 0}}}, - {"block_size", llvm::json::Object{{"type", "integer"}, {"minimum", 0}}}, - {"syndrome_size", - llvm::json::Object{{"type", "integer"}, {"minimum", 0}}}, - {"H_sparse", llvm::json::Object{{"$ref", "#/$defs/sparse_matrix"}}}, - {"O_sparse", llvm::json::Object{{"$ref", "#/$defs/sparse_matrix"}}}, - {"D_sparse", llvm::json::Object{{"$ref", "#/$defs/sparse_matrix"}}}, - {"decoder_custom_args", llvm::json::Object{{"type", "object"}}}, - }; - - // Per-type dispatch of decoder_custom_args, generated from the registry: - // a registered type's args follow its schema; a type with no registered - // schema accepts no args (the parser rejects the section outright). - llvm::json::Array dispatch; - for (const auto &name : names) - dispatch.push_back(llvm::json::Object{ - {"if", llvm::json::Object{{"properties", - llvm::json::Object{ - {"type", - llvm::json::Object{{"const", name}}}}}, - {"required", llvm::json::Array{"type"}}}}, - {"then", llvm::json::Object{ - {"properties", llvm::json::Object{ - {"decoder_custom_args", - llvm::json::Object{ - {"$ref", params_ref(name)}}}}}}}}); - dispatch.push_back(llvm::json::Object{ - {"if", - llvm::json::Object{ - {"properties", - llvm::json::Object{ - {"type", - llvm::json::Object{ - {"not", llvm::json::Object{{"enum", registered_name_array( - names)}}}}}}}, - {"required", llvm::json::Array{"type"}}}}, - {"then", - llvm::json::Object{ - {"properties", - llvm::json::Object{{"decoder_custom_args", - llvm::json::Object{{"maxProperties", 0}}}}}}}}); - - llvm::json::Object defs{ - {"sparse_matrix", - llvm::json::Object{{"type", "array"}, - {"items", llvm::json::Object{{"type", "integer"}, - {"minimum", -1}}}}}, - {"decoder_config", - llvm::json::Object{ - {"type", "object"}, - {"properties", std::move(config_properties)}, - {"required", - llvm::json::Array{"id", "type", "block_size", "syndrome_size", - "H_sparse", "O_sparse", "D_sparse"}}, - {"additionalProperties", false}, - {"allOf", std::move(dispatch)}}}, - {"decoder_params", std::move(decoder_params)}, - }; - - llvm::json::Object root{ - {"$schema", "https://json-schema.org/draft/2020-12/schema"}, - {"title", "CUDA-Q QEC realtime decoding configuration"}, - {"description", - "Validates multi_decoder_config YAML documents. Generated from the " - "decoder parameter schemas registered in this installation, so it " - "reflects the decoder plugins loaded at generation time. Per-schema " - "validate hooks (arbitrary cross-field checks) are not representable " - "in JSON Schema; a document that passes may still be rejected when " - "parsed."}, - {"type", "object"}, - {"properties", - llvm::json::Object{ - {"decoders", - llvm::json::Object{ - {"type", "array"}, - {"items", - llvm::json::Object{{"$ref", "#/$defs/decoder_config"}}}}}}}, - {"required", llvm::json::Array{"decoders"}}, - {"additionalProperties", false}, - {"$defs", std::move(defs)}, - }; - - std::string out; - llvm::raw_string_ostream os(out); - llvm::json::OStream json_out(os, /*IndentSize=*/2); - json_out.value(llvm::json::Value(std::move(root))); - return out; -} - -// Stash a copy for consumers that build their own decoder instances from the -// process-wide configuration -- the decoding-server DeviceCallService plugin -// reads it when CUDAQ_QEC_DECODER_CONFIG is not set (in-process path). -// shared_ptr + mutex: the plugin reads this from the realtime dispatcher -// thread while the application thread may call configure_decoders() again; -// shared ownership keeps the reader's config alive across a concurrent -// replacement. -static std::mutex g_last_multi_decoder_config_mutex; -static std::shared_ptr g_last_multi_decoder_config; - int configure_decoders(multi_decoder_config &config) { CUDA_QEC_INFO("Initializing realtime decoding library with config object"); - { - std::lock_guard lock(g_last_multi_decoder_config_mutex); - g_last_multi_decoder_config = - std::make_shared(config); - } // Publish the decoder configuration so CUDA-Q can inject it into // remote-target job requests. The cudaq integration (ExtraPayloadProvider) is // installed by cudaq-qec at load time; this call is a no-op when cudaq-qec is @@ -695,12 +851,6 @@ int configure_decoders(multi_decoder_config &config) { return cudaq::qec::decoding::host::configure_decoders(config); } -std::shared_ptr -last_configured_multi_decoder_config() { - std::lock_guard lock(g_last_multi_decoder_config_mutex); - return g_last_multi_decoder_config; -} - void log_config(const char *config_str, bool from_file) { const bool dump_config = []() { if (auto *ch = std::getenv("CUDAQ_QEC_DEBUG_DECODER")) diff --git a/libs/qec/lib/realtime/decoding-server-cqr/CMakeLists.txt b/libs/qec/lib/realtime/decoding-server-cqr/CMakeLists.txt index 48a99e9ca..2139662b4 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/CMakeLists.txt +++ b/libs/qec/lib/realtime/decoding-server-cqr/CMakeLists.txt @@ -6,305 +6,63 @@ # the terms of the Apache License 2.0 which accompanies this distribution. # # ============================================================================ # -# Handle both standalone (libs/qec) and full-repo builds. -if(CUDAQX_QEC_STANDALONE_BUILD) - set(_core_include_dir "${CMAKE_SOURCE_DIR}/../core/include") -else() - set(_core_include_dir "${CMAKE_SOURCE_DIR}/libs/core/include") -endif() - -# --------------------------------------------------------------------------- -# Optional: GPU RoCE transport via Hololink Sensor Bridge + DOCA -# -# Library names and DOCA path mirror libs/qec/unittests/utils/CMakeLists.txt -# (the existing hololink_qldpc_graph_decoder_bridge target) so both targets -# find the same libraries from the same build tree. -# --------------------------------------------------------------------------- -set(CUDAQ_GPU_ROCE_AVAILABLE FALSE) - -if(HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR AND CUDAQ_REALTIME_INCLUDE_DIR) - find_library(HOLOLINK_CORE_LIB - NAMES hololink_core - PATHS "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}" - "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}/src/hololink/core" - "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}/lib" - NO_DEFAULT_PATH) - - find_library(GPU_ROCE_TRANSCEIVER_LIB - NAMES gpu_roce_transceiver - PATHS "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}" - "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}/src/hololink/operators/gpu_roce_transceiver" - "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}/lib" - NO_DEFAULT_PATH) - - find_library(HOLOLINK_COMMON_LIB - NAMES hololink - PATHS "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}" - "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}/src/hololink/common" - "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}/lib" - NO_DEFAULT_PATH) - - find_library(BASE_RECEIVER_OP_LIB - NAMES base_receiver_op - PATHS "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}" - "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}/src/hololink/operators" - "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR}/lib" - NO_DEFAULT_PATH) - - find_library(IBVERBS_LIB NAMES ibverbs) - - # DOCA at the standard Mellanox install path. - # Arch-specific lib dir mirrors libs/qec/unittests/utils/CMakeLists.txt. - set(_doca_root "/opt/mellanox/doca") - if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)") - set(_doca_lib_dir "${_doca_root}/lib/x86_64-linux-gnu") - elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)") - set(_doca_lib_dir "${_doca_root}/lib/aarch64-linux-gnu") - else() - set(_doca_lib_dir "${_doca_root}/lib") - endif() - find_library(DOCA_VERBS_LIB doca_verbs PATHS "${_doca_lib_dir}" NO_DEFAULT_PATH) - find_library(DOCA_GPUNETIO_LIB doca_gpunetio PATHS "${_doca_lib_dir}" NO_DEFAULT_PATH) - find_library(DOCA_COMMON_LIB doca_common PATHS "${_doca_lib_dir}" NO_DEFAULT_PATH) - - # hololink_wrapper.h lives in CUDAQ_REALTIME_INCLUDE_DIR. - # Derive the lib search dir from the already-found CUDAQ_REALTIME_LIBRARY - # (set by the parent CMakeLists or the CQR plugin detection above). - if(CUDAQ_REALTIME_LIBRARY) - get_filename_component(_cudaq_rt_lib_dir "${CUDAQ_REALTIME_LIBRARY}" DIRECTORY) - else() - get_filename_component(_cudaq_rt_lib_dir "${CUDAQ_REALTIME_INCLUDE_DIR}/../lib" ABSOLUTE) - endif() - find_library(CUDAQ_REALTIME_BRIDGE_HOLOLINK_LIBRARY - NAMES cudaq-realtime-bridge-hololink - HINTS "${_cudaq_rt_lib_dir}" - NO_DEFAULT_PATH) - - # cudaq-realtime-dispatch: declares cudaq_create_dispatch_graph_regular, - # cudaq_launch_dispatch_graph, cudaq_destroy_dispatch_graph, cudaq_function_entry_t. - find_library(CUDAQ_REALTIME_DISPATCH_LIBRARY - NAMES cudaq-realtime-dispatch - HINTS "${_cudaq_rt_lib_dir}" - NO_DEFAULT_PATH) - - # cudaq-realtime-host-dispatch: required transitive dep of the bridge path. - find_library(CUDAQ_REALTIME_HOST_DISPATCH_LIBRARY - NAMES cudaq-realtime-host-dispatch - HINTS "${_cudaq_rt_lib_dir}" - NO_DEFAULT_PATH) - - if(HOLOLINK_CORE_LIB AND GPU_ROCE_TRANSCEIVER_LIB AND HOLOLINK_COMMON_LIB - AND BASE_RECEIVER_OP_LIB AND DOCA_VERBS_LIB AND DOCA_GPUNETIO_LIB - AND DOCA_COMMON_LIB AND CUDAQ_REALTIME_BRIDGE_HOLOLINK_LIBRARY - AND CUDAQ_REALTIME_DISPATCH_LIBRARY AND CUDAQ_REALTIME_HOST_DISPATCH_LIBRARY) - set(CUDAQ_GPU_ROCE_AVAILABLE TRUE CACHE INTERNAL - "GPU RoCE transport component cudaq-qec-decoding-server-gpuroce available") - # Under CMP0126 (NEW) the CACHE set above does NOT update the normal - # variable initialized FALSE at the top of this file, and the normal - # variable shadows the cache in this scope -- so the component gate - # below would never fire. Set the normal variable explicitly. - set(CUDAQ_GPU_ROCE_AVAILABLE TRUE) - message(STATUS - "cudaq-qec-decoding-server-gpuroce: GPU RoCE ENABLED (core=${HOLOLINK_CORE_LIB})") - else() - message(STATUS - "cudaq-qec-decoding-server: GPU RoCE DISABLED " - "(missing one or more of: hololink_core gpu_roce_transceiver hololink " - "base_receiver_op ibverbs doca_* cudaq-realtime-bridge-hololink " - "cudaq-realtime-dispatch cudaq-realtime-host-dispatch)") - endif() -else() - message(STATUS - "cudaq-qec-decoding-server: GPU RoCE DISABLED " - "(HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR or CUDAQ_REALTIME_INCLUDE_DIR not set)") -endif() - -message(STATUS - "cudaq-qec-decoding-server: CPU RoCE DISABLED (skeleton only; ibverbs implementation pending)") - -# --------------------------------------------------------------------------- -# Core decoding server (always built): transport abstraction, session registry, -# per-decoder worker threads, RPC dispatcher. No dependency on CUDAQ_REALTIME. +# Decoding server (skeleton) for the cudaq-realtime (cqr) host-dispatch path. # -# Deliberately excludes GpuRoceTransceiver: consumers of this library (unit -# tests, the CQR plugin) must stay loadable on driverless machines, and the -# GPU RoCE deps (DOCA, the hololink bridge) require libcuda.so.1 at load -# time. gpu_roce lives in the cudaq-qec-decoding-server-gpuroce component -# below, reached through a weak factory symbol (see DecodingServer.cpp). -# --------------------------------------------------------------------------- -add_library(cudaq-qec-decoding-server STATIC - RoundAccumulator.cpp - DecodingSession.cpp - SessionRegistry.cpp - RpcDispatcher.cpp - DecodingServer.cpp - CpuRoceTransceiver.cpp +# This is the server end of the wire: a cudaq::realtime::DeviceCallService +# that registers CUDAQ_DISPATCH_HOST_CALL handlers for the +# default-route RPCs (enqueue_syndromes / get_corrections / reset_decoder, per +# decoder_server_runtime.md) and drives the configured decoders via the +# cudaq-qec-realtime-decoding host API. A simulated QPU (../simulation-cqr) or, +# eventually, real hardware issues those RPCs over cudaq-realtime. +add_library(cudaq-qec-realtime-decoding-server-cqr SHARED + decoding_server_cqr.cpp ) -target_compile_features(cudaq-qec-decoding-server PUBLIC cxx_std_20) +target_compile_options(cudaq-qec-realtime-decoding-server-cqr + PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden) -target_include_directories(cudaq-qec-decoding-server +target_include_directories(cudaq-qec-realtime-decoding-server-cqr PUBLIC - $ $ - $ + $ $ + PRIVATE + ${CUDAQ_REALTIME_INCLUDE_DIR} + ${CUDAQ_DEVICE_CALL_INCLUDE_DIR} + ${CUDAToolkit_INCLUDE_DIRS} ) -target_link_libraries(cudaq-qec-decoding-server +target_link_options(cudaq-qec-realtime-decoding-server-cqr PRIVATE + $<$:-Wl,--exclude-libs,ALL> +) + +target_link_libraries(cudaq-qec-realtime-decoding-server-cqr PUBLIC - # The decode-server core only needs the decoder interface + logger (in - # cudaq-qec-decoders) and the realtime decoding config -- NOT the heavy - # cudaq-qec (which pulls in libcustabilizer etc.). Keeping cudaq-qec here - # violates the realtime-server dependency-closure contract. - cudaq-qec-decoders cudaq-qec-realtime-decoding PRIVATE - # DecodingSession worker threads pin themselves to their decoder's - # cuda_device_id (cudaSetDevice). - CUDA::cudart + cudaq::cudaq-common + ${CUDAQ_DEVICE_CALL_RUNTIME_LIBRARY} ) -set_target_properties(cudaq-qec-decoding-server PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib - POSITION_INDEPENDENT_CODE ON +set(_server_cqr_build_rpath "${CMAKE_BINARY_DIR}/lib") +set(_server_cqr_install_rpath + "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") +if(_CUDAQ_REALTIME_LIB_DIR) + list(APPEND _server_cqr_build_rpath "${_CUDAQ_REALTIME_LIB_DIR}") + list(APPEND _server_cqr_install_rpath "${_CUDAQ_REALTIME_LIB_DIR}") +endif() +if(_CUDAQ_DEVICE_CALL_RUNTIME_LIB_DIR) + list(APPEND _server_cqr_build_rpath "${_CUDAQ_DEVICE_CALL_RUNTIME_LIB_DIR}") + list(APPEND _server_cqr_install_rpath "${_CUDAQ_DEVICE_CALL_RUNTIME_LIB_DIR}") +endif() + +set_target_properties(cudaq-qec-realtime-decoding-server-cqr PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib + BUILD_RPATH "${_server_cqr_build_rpath}" + INSTALL_RPATH "${_server_cqr_install_rpath}" ) -install(TARGETS cudaq-qec-decoding-server +install(TARGETS cudaq-qec-realtime-decoding-server-cqr COMPONENT qec-lib - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) - -# --------------------------------------------------------------------------- -# Optional GPU RoCE component: GpuRoceTransceiver plus the strong definition -# (GpuRoceFactory.cpp) of the factory that DecodingServer.cpp references -# weakly. Kept out of the core library so core consumers carry no DOCA / -# Hololink / CUDA-driver runtime dependencies -- those .so's need -# libcuda.so.1 at load time, which driverless machines (CI containers) lack. -# -# Consumers must link this WHOLE_ARCHIVE: the only reference to the factory -# is weak, which does not pull archive members on its own. -# --------------------------------------------------------------------------- -if(CUDAQ_GPU_ROCE_AVAILABLE) - add_library(cudaq-qec-decoding-server-gpuroce STATIC - GpuRoceTransceiver.cpp - GpuRoceFactory.cpp - ) - - target_compile_definitions(cudaq-qec-decoding-server-gpuroce - PRIVATE CUDAQ_GPU_ROCE_AVAILABLE) - - target_include_directories(cudaq-qec-decoding-server-gpuroce PRIVATE - "${CUDAQ_REALTIME_INCLUDE_DIR}" - "${_doca_root}/include" - ) - - target_link_libraries(cudaq-qec-decoding-server-gpuroce - PUBLIC - cudaq-qec-decoding-server - "${CUDAQ_REALTIME_BRIDGE_HOLOLINK_LIBRARY}" - "${GPU_ROCE_TRANSCEIVER_LIB}" - "${BASE_RECEIVER_OP_LIB}" - "${HOLOLINK_CORE_LIB}" - "${HOLOLINK_COMMON_LIB}" - "${DOCA_VERBS_LIB}" - "${DOCA_GPUNETIO_LIB}" - "${DOCA_COMMON_LIB}" - $<$:${IBVERBS_LIB}> - CUDA::cudart - # libdoca_gpunetio.so and libcudaq-realtime-bridge-hololink.so reference - # the CUDA Driver API (cu*); consumers of this static lib inherit that - # dep, so link the driver here. CUDA::cuda_driver resolves to the - # toolkit's stubs in driverless build environments (CI containers). - CUDA::cuda_driver - ) - - set_target_properties(cudaq-qec-decoding-server-gpuroce PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib - POSITION_INDEPENDENT_CODE ON - ) - - install(TARGETS cudaq-qec-decoding-server-gpuroce - COMPONENT qec-lib - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - ) - - # Link canary (never executed, not installed, not a test): a static - # archive alone never resolves symbols, and the one real consumer - # (decoding_server's gpu_roce block) is additionally gated on the - # proprietary cudevice archive, which CI does not provision. Without - # this target, CI (HSB + DOCA present, no GPU driver) would compile - # GpuRoceTransceiver.cpp but never validate its link against the - # hololink / DOCA libraries. The strong factory reference pulls the - # component's members, and the component's PUBLIC deps (including the - # CUDA driver stubs) complete the resolution. - add_executable(cudaq-qec-decoding-server-gpuroce-linkcheck - GpuRoceLinkCheck.cpp - ) - target_link_libraries(cudaq-qec-decoding-server-gpuroce-linkcheck PRIVATE - cudaq-qec-decoding-server-gpuroce - ) -endif() - -# --------------------------------------------------------------------------- -# CQR host-dispatch plugin (only when CUDAQ device_call headers are available): -# CqrTransceiver bridges DeviceCallService handler callbacks to ITransceiver, -# then hands off to DecodingServer for session management and decoding. -# --------------------------------------------------------------------------- -if(CUDAQ_QEC_DEVICE_CALL_SERVICE_AVAILABLE) - add_library(cudaq-qec-realtime-decoding-server-cqr SHARED - decoding_server_cqr.cpp - ) - - target_compile_options(cudaq-qec-realtime-decoding-server-cqr - PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden) - - target_include_directories(cudaq-qec-realtime-decoding-server-cqr - PUBLIC - $ - $ - $ - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} - ${CUDAQ_REALTIME_INCLUDE_DIR} - ${CUDAQ_DEVICE_CALL_INCLUDE_DIR} - ${CUDAToolkit_INCLUDE_DIRS} - ) - - target_link_options(cudaq-qec-realtime-decoding-server-cqr PRIVATE - $<$:-Wl,--exclude-libs,libcudaq-qec-decoding-server.a> - ) - - target_link_libraries(cudaq-qec-realtime-decoding-server-cqr - PUBLIC - cudaq-qec-realtime-decoding - PRIVATE - cudaq-qec-decoding-server - cudaq::cudaq-common - ${CUDAQ_DEVICE_CALL_RUNTIME_LIBRARY} - ) - - set(_server_cqr_build_rpath "${CMAKE_BINARY_DIR}/lib") - set(_server_cqr_install_rpath - "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") - if(_CUDAQ_REALTIME_LIB_DIR) - list(APPEND _server_cqr_build_rpath "${_CUDAQ_REALTIME_LIB_DIR}") - list(APPEND _server_cqr_install_rpath "${_CUDAQ_REALTIME_LIB_DIR}") - endif() - if(_CUDAQ_DEVICE_CALL_RUNTIME_LIB_DIR) - list(APPEND _server_cqr_build_rpath "${_CUDAQ_DEVICE_CALL_RUNTIME_LIB_DIR}") - list(APPEND _server_cqr_install_rpath - "${_CUDAQ_DEVICE_CALL_RUNTIME_LIB_DIR}") - endif() - - set_target_properties(cudaq-qec-realtime-decoding-server-cqr PROPERTIES - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib - BUILD_RPATH "${_server_cqr_build_rpath}" - INSTALL_RPATH "${_server_cqr_install_rpath}" - ) - - install(TARGETS cudaq-qec-realtime-decoding-server-cqr - COMPONENT qec-lib - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ) -endif() diff --git a/libs/qec/lib/realtime/decoding-server-cqr/CpuRoceTransceiver.cpp b/libs/qec/lib/realtime/decoding-server-cqr/CpuRoceTransceiver.cpp deleted file mode 100644 index 078294cf6..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/CpuRoceTransceiver.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#include "CpuRoceTransceiver.h" - -#include - -namespace cudaq::qec::decoding_server { - -CpuRoceTransceiver::CpuRoceTransceiver() { - throw std::runtime_error( - "CpuRoceTransceiver: ibverbs/SoftRoCE implementation pending. " - "Use LoopbackTransceiver for development or GpuRoceTransceiver for " - "production Hololink/DOCA deployments."); -} - -RxFrame CpuRoceTransceiver::recv() { - throw std::logic_error( - "CpuRoceTransceiver::recv() called on failed instance"); -} - -void CpuRoceTransceiver::send(const PeerId & /*peer*/, const uint8_t * /*data*/, - size_t /*len*/) { - throw std::logic_error( - "CpuRoceTransceiver::send() called on failed instance"); -} - -void CpuRoceTransceiver::shutdown() {} - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/CpuRoceTransceiver.h b/libs/qec/lib/realtime/decoding-server-cqr/CpuRoceTransceiver.h deleted file mode 100644 index 3449d88c8..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/CpuRoceTransceiver.h +++ /dev/null @@ -1,33 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include "ITransceiver.h" - -namespace cudaq::qec::decoding_server { - -/// CPU RoCE transport skeleton (ibverbs). -/// -/// This class exists as a build-time stub so the server can be compiled with -/// cpu_roce in the config without a link-time error. The constructor throws at -/// runtime until a full ibverbs QP setup is provided. -/// -/// Do NOT guard construction with CUDAQ_CPU_ROCE_AVAILABLE — the class is -/// always compiled so that make_transport() can give a clear runtime error -/// rather than a linker error or a missing-symbol crash. -class CpuRoceTransceiver final : public ITransceiver { -public: - CpuRoceTransceiver(); - - RxFrame recv() override; - void send(const PeerId &peer, const uint8_t *data, size_t len) override; - void shutdown() override; -}; - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/CqrTransceiver.h b/libs/qec/lib/realtime/decoding-server-cqr/CqrTransceiver.h deleted file mode 100644 index 371ee32b0..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/CqrTransceiver.h +++ /dev/null @@ -1,387 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include "ITransceiver.h" -#include "RpcWireFormat.h" -#include "cudaq/realtime/daemon/dispatcher/cudaq_realtime.h" -#include "cudaq/realtime/daemon/dispatcher/dispatch_kernel_launch.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace cudaq::qec::decoding_server { - -namespace detail { - -/// Validated view of the cudaq-realtime enqueue_syndromes request format. -struct CqrEnqueueFrameView { - const cudaq::realtime::RPCHeader *header = nullptr; - uint64_t decoder_id = 0; - uint64_t counter = 0; - uint64_t syndrome_mapping_id = 0; - uint64_t num_syndromes = 0; - uint64_t byte_count = 0; - const uint8_t *packed_bits = nullptr; -}; - -/// Parse an enqueue request only after proving the advertised payload is -/// physically present in the supplied slot. -inline bool parse_cqr_enqueue_frame(const void *rx_slot, std::size_t slot_size, - CqrEnqueueFrameView &out) { - if (!rx_slot || slot_size < sizeof(cudaq::realtime::RPCHeader)) - return false; - - const auto *header = static_cast(rx_slot); - if (header->magic != cudaq::realtime::RPC_MAGIC_REQUEST || - header->function_id != kEnqueueSyndromesFunctionId) - return false; - - const std::size_t physical_payload = - slot_size - sizeof(cudaq::realtime::RPCHeader); - const std::size_t arg_len = header->arg_len; - if (arg_len > physical_payload) - return false; - - const auto *payload = static_cast(rx_slot) + - sizeof(cudaq::realtime::RPCHeader); - std::size_t offset = 0; - auto read_u64 = [&](uint64_t &value) { - if (offset > arg_len || sizeof(uint64_t) > arg_len - offset) - return false; - std::memcpy(&value, payload + offset, sizeof(uint64_t)); - offset += sizeof(uint64_t); - return true; - }; - - CqrEnqueueFrameView parsed; - parsed.header = header; - // arg3 is a std::vector (CUDAQ_TYPE_BIT_PACKED): the realtime - // device_call ABI serialises the 4th u64 as the stdvec array-length prefix, - // i.e. the # of logical elements = # of bits = num_syndromes. The byte count - // is derived (ceil(bits/8)), not carried on the wire. - if (!read_u64(parsed.decoder_id) || !read_u64(parsed.counter) || - !read_u64(parsed.syndrome_mapping_id) || !read_u64(parsed.num_syndromes)) - return false; - - parsed.byte_count = - bit_packed_bytes(static_cast(parsed.num_syndromes)); - if (parsed.num_syndromes == 0 || parsed.num_syndromes > kMaxSyndromeBits || - offset > arg_len || parsed.byte_count > arg_len - offset) - return false; - - parsed.packed_bits = payload + offset; - out = parsed; - return true; -} - -} // namespace detail - -/// Bridges CUDAQ_REALTIME DeviceCallService handler callbacks to ITransceiver. -/// -/// CUDAQ calls handler functions synchronously with (rx_slot, tx_slot, -/// slot_size), all on the SINGLE transport dispatcher thread. -/// -/// Response-bearing calls (get_corrections, reset_decoder): inject() copies -/// rx_slot bytes into an RxFrame, stores the tx_slot pointer keyed by -/// request_id, and blocks until the DecodingSession worker calls send() with -/// the response — at which point send() copies the bytes to tx_slot and -/// unblocks the handler thread so CUDAQ can return. -/// -/// Fire-and-forget calls (enqueue_syndromes): inject() enqueues the frame -/// and writes an immediate ACCEPTED response into tx_slot WITHOUT blocking — -/// blocking the lone dispatcher thread on decoder execution would serialize -/// every decoder's stream behind every other's, defeating the per-session -/// worker parallelism (and nothing would ever unblock it: on_enqueue sends -/// no response). A deferred decoder error is reported at that decoder's -/// next get_corrections. -/// -/// Format translation: the wire frame follows decoder_server_runtime.md -/// (bit-packed syndromes with an explicit uint64 byte-count prefix on the -/// ARRAY_UINT8 argument); inject() validates it and re-frames to the -/// internal EnqueuePayload layout (same fields, no byte-count prefix). -class CqrTransceiver final : public ITransceiver { -public: - /// Called from CUDAQ handler threads for each incoming RPC. - /// Translates the CUDAQ-format payload to our wire format, enqueues an - /// RxFrame, then blocks until DecodingServer sends the response. - void inject(const void *rx_slot, void *tx_slot, std::size_t slot_size, - uint32_t function_id); - - RxFrame recv() override; - void send(const PeerId &peer, const uint8_t *data, std::size_t len) override; - void shutdown() override; - -private: - bool stopped_ = false; - - // Write an immediate RPCResponse (no result payload) into the CUDAQ - // tx_slot: OK acks fire-and-forget calls; error statuses complete blocking - // calls that will never be dispatched (rejects after shutdown). - static void write_ack(void *tx_slot, uint32_t request_id, - uint64_t ptp_timestamp, - RpcStatus status = RpcStatus::OK); - - struct PendingTx { - void *tx_slot; - std::size_t slot_size; - std::promise done; - }; - - std::mutex mtx_; - std::condition_variable cv_; - std::deque inbox_; - std::unordered_map pending_; // keyed by request_id - - // Translate CUDAQ enqueue_syndromes payload (stdvec format) to our - // RPCHeader + EnqueuePayload + bit-packed bytes. - static bool build_enqueue_frame(const void *rx_slot, std::size_t slot_size, - RxFrame &out); - - // For get_corrections and reset_decoder the field layouts are compatible; - // copy rx_slot verbatim after swapping to our magic/RPCHeader type. - static bool build_passthrough_frame(const void *rx_slot, - std::size_t slot_size, uint32_t fn_id, - RxFrame &out); -}; - -// --------------------------------------------------------------------------- -// Inline implementation -// --------------------------------------------------------------------------- - -inline void CqrTransceiver::inject(const void *rx_slot, void *tx_slot, - std::size_t slot_size, - uint32_t function_id) { - if (!rx_slot || !tx_slot || slot_size < sizeof(RPCHeader)) - return; - - RxFrame frame; - bool ok = - (function_id == kEnqueueSyndromesFunctionId) - ? build_enqueue_frame(rx_slot, slot_size, frame) - : build_passthrough_frame(rx_slot, slot_size, function_id, frame); - if (!ok) { - // For blocking calls (get_corrections, reset_decoder) an unwritten tx_slot - // would stall the CUDAQ dispatcher indefinitely. Write BAD_REQUEST so it - // gets a valid magic word regardless of call type. - const auto *cqr = static_cast(rx_slot); - auto *resp = static_cast(tx_slot); - resp->status = static_cast(RpcStatus::BAD_REQUEST); - resp->result_len = 0; - resp->request_id = cqr->request_id; - resp->ptp_timestamp = cqr->ptp_timestamp; - __atomic_store_n(reinterpret_cast(tx_slot), - cudaq::realtime::RPC_MAGIC_RESPONSE, __ATOMIC_RELEASE); - return; - } - - const auto *hdr = reinterpret_cast(frame.buf.data()); - const uint32_t rid = hdr->request_id; - const uint64_t ptp = hdr->ptp_timestamp; // save before frame is moved - - if (function_id == kEnqueueSyndromesFunctionId) { - // Fire-and-forget: hand the frame to the server and ACK immediately - // (status OK = ACCEPTED) -- the dispatcher thread must not park on - // decoder execution, and per the spec the dispatcher still emits an - // RPCResponse into the tx_slot (the transport needs it to complete the - // slot; the caller drops it). A deferred decoder error is reported at - // this decoder's next get_corrections. - { - std::lock_guard lk(mtx_); - inbox_.push_back(std::move(frame)); - } - cv_.notify_one(); - write_ack(tx_slot, rid, ptp); - return; - } - - std::future fut; - { - std::lock_guard lk(mtx_); - // Reject new blocking RPCs after shutdown: the recv loop is exiting and - // will never dispatch this frame, so parking on the promise would hang - // the CUDAQ dispatcher thread forever. Complete the slot immediately. - if (stopped_) { - write_ack(tx_slot, rid, ptp, RpcStatus::BAD_REQUEST); - return; - } - auto &p = pending_[rid]; - p.tx_slot = tx_slot; - p.slot_size = slot_size; - fut = p.done.get_future(); - inbox_.push_back(std::move(frame)); - } - cv_.notify_one(); - - // Block until the DecodingSession worker calls send() with the response. - fut.wait(); -} - -inline RxFrame CqrTransceiver::recv() { - std::unique_lock lk(mtx_); - cv_.wait(lk, [this] { return !inbox_.empty() || stopped_; }); - if (inbox_.empty()) - return {}; // shutdown sentinel (empty buf) - RxFrame frame = std::move(inbox_.front()); - inbox_.pop_front(); - return frame; -} - -inline void CqrTransceiver::shutdown() { - // Move out all in-flight pending entries under the lock, then complete - // them outside it. The recv loop exits without draining inbox_, so a - // frame that inject() already queued would otherwise leave its handler - // thread parked in fut.wait() forever. Write BAD_REQUEST into each - // tx_slot (the CUDAQ transport needs a valid response to complete the - // slot) and fulfill the promise to unblock the waiter. - std::unordered_map drained; - { - std::lock_guard lk(mtx_); - stopped_ = true; - drained = std::move(pending_); - pending_.clear(); - } - cv_.notify_all(); - for (auto &[rid, p] : drained) { - write_ack(p.tx_slot, rid, /*ptp_timestamp=*/0, RpcStatus::BAD_REQUEST); - p.done.set_value(); - } -} - -inline void CqrTransceiver::write_ack(void *tx_slot, uint32_t request_id, - uint64_t ptp_timestamp, - RpcStatus status) { - auto *resp = static_cast(tx_slot); - resp->status = static_cast(status); - resp->result_len = 0; - resp->request_id = request_id; - resp->ptp_timestamp = ptp_timestamp; - // Publish the magic last (release store) so the CUDAQ runtime sees a - // complete response before observing the magic word. - __atomic_store_n(reinterpret_cast(tx_slot), - cudaq::realtime::RPC_MAGIC_RESPONSE, __ATOMIC_RELEASE); -} - -inline void CqrTransceiver::send(const PeerId & /*peer*/, const uint8_t *data, - std::size_t len) { - if (!data || len < sizeof(RPCResponse)) - return; - - const auto *resp = reinterpret_cast(data); - const uint32_t rid = resp->request_id; - - std::lock_guard lk(mtx_); - auto it = pending_.find(rid); - if (it == pending_.end()) - return; - - auto &p = it->second; - if (len > p.slot_size) { - // Truncating would leave result_len advertising bytes that were never - // written, so the client would read stale slot memory as correction - // bits. Fail the RPC explicitly instead (the pre-decoding-server code - // returned result-buffer-too-small here). - write_ack(p.tx_slot, rid, resp->ptp_timestamp, RpcStatus::INTERNAL_ERROR); - p.done.set_value(); - pending_.erase(it); - return; - } - - // Write our RPCResponse into the CUDAQ tx_slot (layouts are compatible). - std::memcpy(p.tx_slot, data, len); - // Publish the magic last (release store) so the CUDAQ runtime sees a - // complete response before observing the magic word. - __atomic_store_n(reinterpret_cast(p.tx_slot), - cudaq::realtime::RPC_MAGIC_RESPONSE, __ATOMIC_RELEASE); - - p.done.set_value(); - pending_.erase(it); -} - -inline bool CqrTransceiver::build_enqueue_frame(const void *rx_slot, - std::size_t slot_size, - RxFrame &out) { - // Spec 5-arg wire format (decoder_server_runtime.md): - // [u64 decoder_id][u64 counter][u64 syndrome_mapping_id] - // [u64 num_syndromes][u64 array_len][u8 x ceil(bits/8) (bit-packed)] - // syndrome_bits is a std::vector (CUDAQ_TYPE_BIT_PACKED); CUDAQ - // serialises it as [u64 array_len = # logical bits][packed bytes]. - detail::CqrEnqueueFrameView request; - if (!detail::parse_cqr_enqueue_frame(rx_slot, slot_size, request)) - return false; - - // Re-frame to RPCHeader + EnqueuePayload + bit-packed bytes (the internal - // layout drops the byte-count prefix; the bits stay packed as-is). - out.buf.resize(sizeof(RPCHeader) + sizeof(EnqueuePayload) + - request.byte_count); - - auto *hdr = reinterpret_cast(out.buf.data()); - hdr->magic = kRPCRequestMagic; - hdr->function_id = kEnqueueSyndromesFunctionId; - hdr->arg_len = - static_cast(sizeof(EnqueuePayload) + request.byte_count); - hdr->request_id = request.header->request_id; - hdr->ptp_timestamp = request.header->ptp_timestamp; - - auto *req = - reinterpret_cast(out.buf.data() + sizeof(RPCHeader)); - req->decoder_id = static_cast(request.decoder_id); - req->counter = static_cast(request.counter); - req->syndrome_mapping_id = static_cast(request.syndrome_mapping_id); - req->num_syndromes = static_cast(request.num_syndromes); - - uint8_t *dst = out.buf.data() + sizeof(RPCHeader) + sizeof(EnqueuePayload); - std::memcpy(dst, request.packed_bits, request.byte_count); - - out.vp_id = 0; - return true; -} - -inline bool CqrTransceiver::build_passthrough_frame(const void *rx_slot, - std::size_t slot_size, - uint32_t fn_id, - RxFrame &out) { - if (slot_size < sizeof(cudaq::realtime::RPCHeader)) - return false; - - const auto *cqr_hdr = - static_cast(rx_slot); - if (cqr_hdr->magic != cudaq::realtime::RPC_MAGIC_REQUEST) - return false; - - // get_corrections and reset_decoder payloads are field-compatible with our - // GetCorrectionsPayload / ResetPayload; copy verbatim and rewrite the header. - const std::size_t total = - sizeof(cudaq::realtime::RPCHeader) + cqr_hdr->arg_len; - if (total > slot_size) - return false; - - out.buf.resize(sizeof(RPCHeader) + cqr_hdr->arg_len); - auto *hdr = reinterpret_cast(out.buf.data()); - hdr->magic = kRPCRequestMagic; - hdr->function_id = fn_id; - hdr->arg_len = cqr_hdr->arg_len; - hdr->request_id = cqr_hdr->request_id; - hdr->ptp_timestamp = cqr_hdr->ptp_timestamp; - - if (cqr_hdr->arg_len > 0) - std::memcpy(out.buf.data() + sizeof(RPCHeader), - static_cast(rx_slot) + - sizeof(cudaq::realtime::RPCHeader), - cqr_hdr->arg_len); - out.vp_id = 0; - return true; -} - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp b/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp deleted file mode 100644 index 5e3a7a8a8..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp +++ /dev/null @@ -1,337 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#include "DecodingServer.h" -#include "CpuRoceTransceiver.h" - -#include "cudaq/qec/logger.h" -#include "cudaq/qec/realtime/decoding_config.h" - -#include -#include -#include -#include -#include -#include -#include - -// GPU RoCE support is an optional component (cudaq-qec-decoding-server-gpuroce) -// so this core library carries no DOCA / Hololink / CUDA-driver dependencies: -// those .so's require libcuda.so.1 at load time, which core consumers (unit -// tests, the CQR plugin) must not impose on driverless machines. Binaries -// that want the gpu_roce transport link the component WHOLE_ARCHIVE, whose -// GpuRoceFactory.cpp provides the strong definition of this factory; anywhere -// else the weak reference is null and make_transport throws. -extern "C" __attribute__((weak)) cudaq::qec::decoding_server::ITransceiver * -cudaqx_qec_make_gpu_roce_transceiver(int pinned_cuda_device); - -namespace cudaq::qec::decoding_server { - -using cudaq::qec::decoding::config::DecoderTransport; - -// --------------------------------------------------------------------------- -// Constructors -// --------------------------------------------------------------------------- - -/// Resolve the CUDA device a decode pipeline runs on from the decoder's -/// cuda_device_id pin; an unpinned decoder (-1) defaults to device 0. The -/// gpu_roce path relies on this to place its rings, dispatch scheduler, and -/// device-side graph fire on the one GPU the FPGA/NIC is affine to -- CUDA -/// graphs cannot split capture and launch across devices, so the decoder must -/// be pinned to that device. -int resolve_decode_device(int decoder_pin) { - return decoder_pin >= 0 ? decoder_pin : 0; -} - -std::unique_ptr -DecodingServer::make_transport(DecoderTransport transport_type, - int pinned_cuda_device) { - switch (transport_type) { - case DecoderTransport::gpu_roce: - // gpu_roce lives in the cudaq-qec-decoding-server-gpuroce component, - // reached through the weak factory. The device is the decoder's - // cuda_device_id pin, resolved inside the factory where GpuRoceConfig - // lives; we just thread the pin to it. - if (cudaqx_qec_make_gpu_roce_transceiver) - return std::unique_ptr( - cudaqx_qec_make_gpu_roce_transceiver(pinned_cuda_device)); - throw std::runtime_error( - "gpu_roce transport requested but GPU RoCE support is not linked into " - "this binary. Build with HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR and DOCA " - "libs, and link cudaq-qec-decoding-server-gpuroce (whole-archive)."); - - case DecoderTransport::cpu_roce: - // CpuRoceTransceiver constructor always throws (ibverbs pending). - return std::make_unique(); - } - throw std::runtime_error("make_transport: unknown DecoderTransport value"); -} - -DecodingServer::DecodingServer(const std::string &config_yaml) { - // Parse the YAML once: SessionRegistry validates the decoder entries - // (including the uniform-transport rule — MVP limitation: heterogeneous - // deployments require per-session transceiver binding, deferred to a - // follow-up once CpuRoce/GpuRoceTransceiverAdapter are available) and - // required_transport() then drives transceiver creation. - std::ifstream f(config_yaml); - if (!f.is_open()) - throw std::runtime_error("Cannot open config: " + config_yaml); - std::string yaml_str((std::istreambuf_iterator(f)), {}); - auto config = - cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( - yaml_str); - if (config.decoders.empty()) - throw std::runtime_error("No decoders in config: " + config_yaml); - registry_.load_from_config(config, config_yaml); - register_handlers(); - - const auto transport_type = registry_.required_transport(); - // gpu_roce must run on the GPU the FPGA/NIC is affine to; when exactly one - // session is booting, pass its decoder's cuda_device_id so the factory can - // place the transport on that device. - const auto &boot_sessions = registry_.sessions(); - const int pinned_cuda_device = - boot_sessions.size() == 1 - ? boot_sessions.begin()->second->dec->get_cuda_device_id() - : -1; - auto t = make_transport(transport_type, pinned_cuda_device); - ITransceiver *raw = t.get(); - owned_transports_.push_back(std::move(t)); - function_transport_[kEnqueueSyndromesFunctionId] = raw; - function_transport_[kGetCorrectionsFunctionId] = raw; - function_transport_[kResetDecoderFunctionId] = raw; - - // For the GPU RoCE path, wire the first (and only) session's decoder graph - // to the Hololink ring buffer via the CUDAQ device-graph scheduler. - // Multi-decoder GPU RoCE binding is deferred to a follow-up. - if (transport_type == DecoderTransport::gpu_roce) { - const auto &sessions = registry_.sessions(); - if (sessions.size() != 1) - throw std::runtime_error( - "GPU RoCE transport currently supports exactly one decoder session; " - "found " + - std::to_string(sessions.size()) + - ". Multi-decoder GPU RoCE is deferred."); - auto *session = sessions.begin()->second.get(); - if (!session->graph_resources) - throw std::runtime_error( - "GPU RoCE requires a decoder that supports graph dispatch " - "(supports_graph_dispatch() must return true and " - "capture_decode_graph() must succeed)"); - if (!raw->launch_device_scheduler(session->graph_resources.get())) - throw std::runtime_error( - "gpu_roce transceiver did not provide a device scheduler"); - } -} - -DecodingServer::DecodingServer(std::unique_ptr transport, - const std::string &config_yaml) { - ITransceiver *raw = transport.get(); - owned_transports_.push_back(std::move(transport)); - function_transport_[kEnqueueSyndromesFunctionId] = raw; - function_transport_[kGetCorrectionsFunctionId] = raw; - function_transport_[kResetDecoderFunctionId] = raw; - try { - init(config_yaml); - } catch (...) { - registry_.stop_workers(); - throw; - } -} - -DecodingServer::DecodingServer( - std::unique_ptr transport, - const cudaq::qec::decoding::config::multi_decoder_config &config) { - ITransceiver *raw = transport.get(); - owned_transports_.push_back(std::move(transport)); - function_transport_[kEnqueueSyndromesFunctionId] = raw; - function_transport_[kGetCorrectionsFunctionId] = raw; - function_transport_[kResetDecoderFunctionId] = raw; - try { - registry_.load_from_config(config, "configure_decoders()"); - } catch (...) { - // Members destroy in reverse order (transports before registry); join any - // already-started workers while the transports still exist. - registry_.stop_workers(); - throw; - } - register_handlers(); -} - -DecodingServer::DecodingServer(std::vector> owned, - TransportMap function_transport, - const std::string &config_yaml) - : owned_transports_(std::move(owned)), - function_transport_(std::move(function_transport)) { - try { - init(config_yaml); - } catch (...) { - registry_.stop_workers(); - throw; - } -} - -DecodingServer::~DecodingServer() { - stop(); - // Join session workers while owned_transports_ is still alive: queued - // WorkItems reply via raw ITransceiver pointers. Decoder/graph teardown - // still happens in ~registry_, after the transports, per the member-order - // comment in DecodingServer.h. - registry_.stop_workers(); -} - -// --------------------------------------------------------------------------- -// init — load sessions and register RPC handlers -// --------------------------------------------------------------------------- - -void DecodingServer::init(const std::string &config_yaml) { - registry_.load_from_config(config_yaml); - register_handlers(); -} - -void DecodingServer::register_handlers() { - // enqueue_syndromes — fire-and-forget at the RPC level; the transport - // layer ACKs delivery (ACCEPTED), and a queue-full drop is reported both - // here and at the next get_corrections. - dispatcher_.register_handler( - kEnqueueSyndromesFunctionId, - [this](RxFrame frame, ResponseWriter &writer) { - if (frame.buf.size() < sizeof(RPCHeader) + sizeof(EnqueuePayload)) { - writer.write_error(RpcStatus::BAD_REQUEST); - return; - } - const auto *req = reinterpret_cast( - frame.buf.data() + sizeof(RPCHeader)); - const auto *hdr = reinterpret_cast(frame.buf.data()); - - auto &session = registry_.get(static_cast(req->decoder_id)); - - WorkItem item; - item.function_id = kEnqueueSyndromesFunctionId; - item.frame_buf = std::move(frame.buf); - item.peer = frame.peer; - item.request_id = hdr->request_id; - item.ptp_timestamp = hdr->ptp_timestamp; - item.vp_id = frame.vp_id; - item.response_transport = writer.transport(); - item.release_fn = std::move(frame.release_fn); - - if (!session.try_enqueue(std::move(item))) { - session.latch_syndromes_dropped(); - writer.write_error(RpcStatus::SYNDROMES_DROPPED); - } - }); - - // get_corrections — response sent by the worker thread. - dispatcher_.register_handler( - kGetCorrectionsFunctionId, [this](RxFrame frame, ResponseWriter &writer) { - if (frame.buf.size() < - sizeof(RPCHeader) + sizeof(GetCorrectionsPayload)) { - writer.write_error(RpcStatus::BAD_REQUEST); - return; - } - const auto *req = reinterpret_cast( - frame.buf.data() + sizeof(RPCHeader)); - const auto *hdr = reinterpret_cast(frame.buf.data()); - - auto &session = registry_.get(static_cast(req->decoder_id)); - - WorkItem item; - item.function_id = kGetCorrectionsFunctionId; - item.frame_buf = std::move(frame.buf); - item.peer = frame.peer; - item.request_id = hdr->request_id; - item.ptp_timestamp = hdr->ptp_timestamp; - item.vp_id = frame.vp_id; - item.response_transport = writer.transport(); - - if (!session.try_enqueue(std::move(item))) - writer.write_error(RpcStatus::BUSY); - }); - - // reset_decoder — response sent by the worker thread. - dispatcher_.register_handler( - kResetDecoderFunctionId, [this](RxFrame frame, ResponseWriter &writer) { - if (frame.buf.size() < sizeof(RPCHeader) + sizeof(ResetPayload)) { - writer.write_error(RpcStatus::BAD_REQUEST); - return; - } - const auto *req = reinterpret_cast( - frame.buf.data() + sizeof(RPCHeader)); - const auto *hdr = reinterpret_cast(frame.buf.data()); - - auto &session = registry_.get(static_cast(req->decoder_id)); - - WorkItem item; - item.function_id = kResetDecoderFunctionId; - item.frame_buf = std::move(frame.buf); - item.peer = frame.peer; - item.request_id = hdr->request_id; - item.ptp_timestamp = hdr->ptp_timestamp; - item.vp_id = frame.vp_id; - item.response_transport = writer.transport(); - - if (!session.try_enqueue(std::move(item))) - writer.write_error(RpcStatus::BUSY); - }); -} // register_handlers - -// --------------------------------------------------------------------------- -// run / stop -// --------------------------------------------------------------------------- - -void DecodingServer::run() { - std::vector unique_transports; - for (auto &[fid, t] : function_transport_) { - if (std::find(unique_transports.begin(), unique_transports.end(), t) == - unique_transports.end()) - unique_transports.push_back(t); - } - - CUDA_QEC_INFO("DecodingServer: starting {} receiver thread(s)", - unique_transports.size()); - - // All threads share dispatcher_ — routing is by function_id, not transport. - std::vector recv_threads; - recv_threads.reserve(unique_transports.size()); - for (ITransceiver *t : unique_transports) { - recv_threads.emplace_back([this, t] { - while (!shutdown_.load(std::memory_order_acquire)) { - RxFrame frame = t->recv(); - if (frame.buf.empty()) - continue; // shutdown sentinel; loop re-checks the flag - dispatcher_.dispatch(std::move(frame), *t); - } - }); - } - - for (auto &th : recv_threads) - th.join(); - - CUDA_QEC_INFO("DecodingServer: all receiver threads exited"); -} - -void DecodingServer::print_session_stats() const { - for (const auto &[id, session] : registry_.sessions()) { - std::cout << "QEC_DECODING_SERVER_DECODER_STATS id=" << id - << " decodes=" << session->decode_count.load() - << " enqueues=" << session->enqueue_count.load() - << " corrections=" << session->get_corrections_count.load() - << " resets=" << session->reset_count.load() - << " errors=" << session->error_count.load() << std::endl; - } -} - -void DecodingServer::stop() { - shutdown_.store(true, std::memory_order_release); - // Unblock any receive loop parked in recv(). - for (auto &t : owned_transports_) - t->shutdown(); -} - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h b/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h deleted file mode 100644 index 0cab850bf..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h +++ /dev/null @@ -1,103 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include "ITransceiver.h" -#include "RpcDispatcher.h" -#include "SessionRegistry.h" -#include "cudaq/qec/realtime/decoding_config.h" - -#include -#include -#include -#include -#include - -namespace cudaq::qec::decoding_server { - -/// Resolve the CUDA device a decode pipeline runs on from the decoder's -/// cuda_device_id (-1 when unpinned). An unpinned decoder defaults to device 0. -int resolve_decode_device(int decoder_pin); - -/// Maps function_id → non-owning ITransceiver pointer. -/// Ownership lives in DecodingServer::owned_transports_. -using TransportMap = std::unordered_map; - -/// Top-level server: owns the registry and dispatcher, holds the -/// transceiver(s), and runs the blocking receive loop. -class DecodingServer { -public: - /// Config-driven constructor: reads the transport type from \p config_yaml - /// and creates the appropriate transceiver. Requires CUDAQ_REALTIME for - /// RoCE transports; throws std::runtime_error if the adapters are not - /// available. Use the explicit-transceiver constructors for testing with - /// LoopbackTransceiver. - explicit DecodingServer(const std::string &config_yaml); - - /// Single-transceiver constructor: all three RPCs share one transport. - DecodingServer(std::unique_ptr transport, - const std::string &config_yaml); - - /// Single-transceiver constructor from an already-parsed config -- the - /// in-process path where the application handed the config to - /// configure_decoders() rather than pointing at a YAML file. - DecodingServer( - std::unique_ptr transport, - const cudaq::qec::decoding::config::multi_decoder_config &config); - - /// Split-transport constructor: each function_id dispatched to its own - /// transceiver. \p owned is moved in; \p function_transport holds raw - /// pointers into \p owned. - DecodingServer(std::vector> owned, - TransportMap function_transport, - const std::string &config_yaml); - - /// Stops the transports and joins all session workers before any member is - /// destroyed: workers drain queued items that reply through raw - /// ITransceiver pointers into owned_transports_, so they must finish while - /// the transports are still alive. - ~DecodingServer(); - - /// Block until stop() is called. - void run(); - - /// Thread-safe; signals the receive loop to exit after the current frame. - void stop(); - - /// Print one QEC_DECODING_SERVER_DECODER_STATS line per session to stdout - /// (test/diagnostic evidence; callers gate on QEC_DECODING_SERVER_STATS). - void print_session_stats() const; - -private: - void init(const std::string &config_yaml); - void register_handlers(); - - /// Create a transceiver for \p transport_type. Throws for RoCE transports - /// until CpuRoceTransceiverAdapter / GpuRoceTransceiverAdapter are - /// available via CUDAQ_REALTIME. - static std::unique_ptr - make_transport(cudaq::qec::decoding::config::DecoderTransport transport_type, - int pinned_cuda_device); - - // Destruction order matters: the GPU RoCE scheduler (inside - // owned_transports_) holds a cudaGraphExec_t captured from a session's - // decoder. The scheduler must be destroyed (cudaStreamSynchronize + - // cudaq_destroy_dispatch_graph) before registry_ releases the decoder and its - // graph resources. C++ destroys members in reverse declaration order, so - // registry_ must be declared BEFORE owned_transports_. - SessionRegistry registry_; - RpcDispatcher dispatcher_; - std::atomic shutdown_{false}; - /// Maps function_id → transceiver; used to deduplicate receiver threads. - /// Routing within the server is by function_id, not by decoder_id. - TransportMap function_transport_; - std::vector> owned_transports_; -}; - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp b/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp deleted file mode 100644 index a12d5b541..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp +++ /dev/null @@ -1,395 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#include "DecodingSession.h" -#include "DecodingServer.h" -#include "RpcWireFormat.h" -#include "../../hardware_guards.h" -#include "cudaq/qec/logger.h" - -#include -#include -#include -#include -#include - -namespace cudaq::qec::decoding_server { - -namespace { - -void set_graph_capture_device(const cudaq::qec::decoder &decoder) { - const int device = resolve_decode_device(decoder.get_cuda_device_id()); - cudaq::qec::detail_affinity::set_cuda_device_for_decode(device); - if (device >= 0) - CUDA_QEC_INFO( - "DecodingSession::create: set CUDA device {} before graph capture", - device); -} - -} // namespace - -// Busy high-water mark across all sessions (worker threads increment while -// executing an item). -static std::atomic g_busy_sessions{0}; -static std::atomic g_max_busy_sessions{0}; - -uint64_t max_concurrent_busy_sessions() { - return g_max_busy_sessions.load(std::memory_order_relaxed); -} - -DecodingSession::~DecodingSession() { stop_worker(); } - -void DecodingSession::stop_worker() { - { - // The store must happen under queue_mutex: worker_loop's untimed wait - // checks the flag under the same lock, so this serializes against the - // predicate-check-then-block window and the notify cannot be lost. - std::lock_guard lk(queue_mutex); - shutdown.store(true, std::memory_order_release); - } - queue_cv.notify_one(); - if (worker.joinable()) - worker.join(); -} - -std::unique_ptr -DecodingSession::create(std::unique_ptr decoder, - SyndromeMappingTable mapping_table_arg) { - if (!decoder) - throw std::invalid_argument("DecodingSession requires a decoder"); - - auto s = std::make_unique(); - s->dec = std::move(decoder); - - if (s->dec->supports_graph_dispatch()) { - set_graph_capture_device(*s->dec); - void *gr = s->dec->capture_decode_graph(); - s->graph_resources = - GraphResourcesPtr(gr, GraphResourcesDeleter{s->dec.get()}); - } - - s->mapping_table = std::move(mapping_table_arg); - return s; -} - -void DecodingSession::start_worker() { - // The pin must happen ON the worker thread (CUDA device selection is - // thread-local), but a failure is a startup error that belongs to the - // caller: hand it back through a promise so load_from_config aborts the - // server instead of a worker silently decoding on the wrong device. - std::promise pinned; - auto pin_result = pinned.get_future(); - worker = std::thread([this, &pinned] { - try { - cudaq::qec::detail_affinity::set_cuda_device_for_decode( - dec->get_cuda_device_id()); - pinned.set_value(); - } catch (...) { - pinned.set_exception(std::current_exception()); - return; // never serve work from a mispinned thread - } - worker_loop(); - }); - try { - pin_result.get(); - } catch (...) { - if (worker.joinable()) - worker.join(); - throw; - } -} - -bool DecodingSession::try_enqueue(WorkItem item) { - std::lock_guard lk(queue_mutex); - if (work_queue.size() >= queue_depth) { - ++busy_count; - return false; - } - work_queue.push(std::move(item)); - queue_cv.notify_one(); - return true; -} - -void DecodingSession::latch_syndromes_dropped() { - syndromes_dropped.store(true, std::memory_order_release); - ++syndromes_dropped_count; -} - -static void send_response(ITransceiver &transport, const PeerId &peer, - uint32_t request_id, uint64_t ptp_timestamp, - RpcStatus status, - const uint8_t *result_data = nullptr, - size_t result_len = 0) { - std::vector buf(sizeof(RPCResponse) + result_len); - auto *hdr = reinterpret_cast(buf.data()); - hdr->magic = kRPCResponseMagic; - hdr->status = static_cast(status); - hdr->result_len = static_cast(result_len); - hdr->request_id = request_id; - hdr->ptp_timestamp = ptp_timestamp; - if (result_data && result_len) - std::memcpy(buf.data() + sizeof(RPCResponse), result_data, result_len); - transport.send(peer, buf.data(), buf.size()); -} - -// Uses item.response_transport so split-transport sessions reply on the correct -// transport. -void DecodingSession::on_enqueue(const WorkItem &item) { - ++enqueue_count; - // No manual release_fn handling: WorkItem::release_fn is a ReleaseFn that - // fires when the item is destroyed at the end of the worker-loop iteration, - // covering every early return and the exception path. - - // Once an enqueue has been dropped or processing has failed, accepting more - // fragments would make the shot's measurement history unknowable. Only a - // full reset can establish a clean epoch again. - if (syndromes_dropped.load(std::memory_order_acquire) || - shot_state == ShotState::failed) - return; - - const size_t min_size = sizeof(RPCHeader) + sizeof(EnqueuePayload); - if (item.frame_buf.size() < min_size) { - ++error_count; - shot_state = ShotState::failed; - return; // enqueue_syndromes never sends a response - } - - const auto *req = reinterpret_cast( - item.frame_buf.data() + sizeof(RPCHeader)); - - // enqueue_syndromes is fire-and-forget: the caller already received the - // transport-level ACK, so a response here would be unsolicited — silently - // dropped on CQR (no pending_ entry) and protocol-desynchronizing on - // in-order transports. Latch the failure; it surfaces as INTERNAL_ERROR - // at this decoder's next get_corrections. - if (req->num_syndromes <= 0 || - static_cast(req->num_syndromes) > kMaxSyndromeBits) { - ++error_count; - shot_state = ShotState::failed; - return; - } - const size_t syndrome_bytes = - bit_packed_bytes(static_cast(req->num_syndromes)); - if (item.frame_buf.size() < min_size + syndrome_bytes) { - ++error_count; - shot_state = ShotState::failed; - return; - } - - const uint8_t *bit_data = - item.frame_buf.data() + sizeof(RPCHeader) + sizeof(EnqueuePayload); - - // TODO: add byte-packed compat path once compiler lowering PR lands. - // Unpack bit-packed syndromes to byte-per-bit for the decoder. - std::vector unpacked(static_cast(req->num_syndromes)); - for (int64_t i = 0; i < req->num_syndromes; ++i) - unpacked[i] = (bit_data[i / 8] >> (i % 8)) & 1u; - - RoundKey key{ - .decoder_id = static_cast(req->decoder_id), - .counter = static_cast(req->counter), - .syndrome_mapping_id = static_cast(req->syndrome_mapping_id), - }; - - try { - // Any accepted input after a completed decode starts a new volume; the old - // correction vector must not be reported as the result of that volume. - shot_state = ShotState::collecting; - auto completed = accumulator.ingest(key, item.vp_id, unpacked.data(), - unpacked.size(), mapping_table); - if (!completed) - return; - - const size_t expected_syndromes = dec->get_num_msyn_per_decode(); - if (accepted_syndromes > expected_syndromes || - completed->bits.size() > expected_syndromes - accepted_syndromes) - throw std::invalid_argument( - "Syndrome volume exceeds decoder measurement capacity"); - - accepted_syndromes += completed->bits.size(); - // Host-decoder path (CQR / Loopback transports). On the gpu_roce path, - // the CUDAQ device-graph scheduler (cudaq_create_dispatch_graph_regular) - // handles RX→dispatch→decode→TX entirely on the GPU; this worker thread - // is never reached for GPU RoCE sessions. - const bool did_decode = - dec->enqueue_syndrome(completed->bits.data(), completed->bits.size()); - - if (did_decode) { - ++decode_count; - accepted_syndromes = 0; - shot_state = ShotState::result_ready; - } - } catch (const std::exception &e) { - cudaq::qec::error("DecodingSession::on_enqueue: {}", e.what()); - ++error_count; - // Fire-and-forget: no response carries this failure, so latch it and - // surface it until the client establishes a clean epoch with reset. - shot_state = ShotState::failed; - } -} - -void DecodingSession::on_get_corrections(const WorkItem &item) { - ++get_corrections_count; - - if (item.frame_buf.size() < - sizeof(RPCHeader) + sizeof(GetCorrectionsPayload)) { - ++error_count; - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::BAD_REQUEST); - return; - } - - const auto *req = reinterpret_cast( - item.frame_buf.data() + sizeof(RPCHeader)); - - // Spec validation: return_size (the OUT std::vector length) must be - // positive. - if (req->return_size <= 0) { - ++error_count; - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::BAD_REQUEST); - return; - } - - if (syndromes_dropped.load(std::memory_order_acquire)) { - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::SYNDROMES_DROPPED); - return; - } - - // Surface a sticky deferred enqueue failure from this shot. Reporting it - // does not make partially accumulated decoder state safe to reuse. - if (shot_state == ShotState::failed) { - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::INTERNAL_ERROR); - return; - } - - try { - const auto return_size = static_cast(req->return_size); - if (return_size != dec->get_num_observables()) { - ++error_count; - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::BAD_REQUEST); - return; - } - if (shot_state != ShotState::result_ready) { - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::NOT_READY); - return; - } - const uint8_t *corrections = dec->get_obs_corrections(); - if (!corrections) { - shot_state = ShotState::failed; - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::INTERNAL_ERROR); - return; - } - // result_len = ceil(R/8) exactly per decoder_server_runtime.md spec. - // The spec forbids trailing padding in the wire result_len; if a transport - // layer needs 8-byte alignment, it must add padding in its own framing. - const size_t result_len = bit_packed_bytes(return_size); - // get_obs_corrections() returns byte-per-bit; pack into the wire format. - std::vector packed(result_len, 0); - for (size_t i = 0; i < return_size; ++i) { - if (corrections[i] & 1u) - packed[i / 8] |= static_cast(1u << (i % 8)); - } - if (req->reset) { - // clear_corrections (not a full reset_decoder): matches the host-path - // semantics of get_corrections(reset=true). Runs BEFORE the OK is - // sent: `packed` already owns a copy of the correction bits, and a - // throw here must produce the single INTERNAL_ERROR response below, - // not a second response after an already-delivered OK. - dec->clear_corrections(); - shot_state = ShotState::collecting; - } - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::OK, packed.data(), result_len); - } catch (const std::exception &e) { - cudaq::qec::error("DecodingSession::on_get_corrections: {}", e.what()); - ++error_count; - shot_state = ShotState::failed; - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::INTERNAL_ERROR); - } -} - -void DecodingSession::on_reset(const WorkItem &item) { - ++reset_count; - try { - dec->reset_decoder(); - accumulator.clear(); - syndromes_dropped.store(false, std::memory_order_release); - accepted_syndromes = 0; - shot_state = ShotState::collecting; - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::OK); - } catch (const std::exception &e) { - cudaq::qec::error("DecodingSession::on_reset: {}", e.what()); - ++error_count; - shot_state = ShotState::failed; - send_response(*item.response_transport, item.peer, item.request_id, - item.ptp_timestamp, RpcStatus::INTERNAL_ERROR); - } -} - -void DecodingSession::worker_loop() { - while (true) { - WorkItem item; - { - std::unique_lock lk(queue_mutex); - // Untimed wait: stop_worker() stores the shutdown flag under - // queue_mutex before notifying, so the wakeup cannot be lost and no - // 100 ms poll is needed. - queue_cv.wait(lk, [this] { - return !work_queue.empty() || shutdown.load(std::memory_order_acquire); - }); - - if (work_queue.empty()) - break; // woken by stop_worker() with nothing left to drain - - item = std::move(work_queue.front()); - work_queue.pop(); - } - - const uint64_t busy = - g_busy_sessions.fetch_add(1, std::memory_order_relaxed) + 1; - uint64_t observed = g_max_busy_sessions.load(std::memory_order_relaxed); - while (busy > observed && !g_max_busy_sessions.compare_exchange_weak( - observed, busy, std::memory_order_relaxed)) - ; - - // Last-resort containment: an exception escaping the worker thread would - // std::terminate the whole process. The handlers catch std::exception - // internally, but allocations outside their try blocks (e.g. the unpacked - // syndrome vector) and non-std exceptions from decoder plugins would - // otherwise escape. - try { - if (item.function_id == kEnqueueSyndromesFunctionId) - on_enqueue(item); - else if (item.function_id == kGetCorrectionsFunctionId) - on_get_corrections(item); - else if (item.function_id == kResetDecoderFunctionId) - on_reset(item); - } catch (const std::exception &e) { - cudaq::qec::error("DecodingSession worker: unhandled exception: {}", - e.what()); - ++error_count; - shot_state = ShotState::failed; - } catch (...) { - cudaq::qec::error("DecodingSession worker: unhandled non-std exception"); - ++error_count; - shot_state = ShotState::failed; - } - - g_busy_sessions.fetch_sub(1, std::memory_order_relaxed); - } -} - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.h b/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.h deleted file mode 100644 index ad100e3e0..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.h +++ /dev/null @@ -1,141 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include "ITransceiver.h" -#include "RoundAccumulator.h" -#include "cudaq/qec/decoder.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cudaq::qec::decoding_server { - -/// A unit of work dispatched from the RpcDispatcher to a DecodingSession worker -/// thread. The payload is an owned copy of the full frame bytes so that the -/// dispatcher can return the transport ring slot immediately after dispatch. -/// -/// release_fn: propagated from RxFrame::release_fn. It fires when the -/// WorkItem is destroyed — after the worker has processed it, or on any drop -/// path (queue full, validation failure). On CPU/CQR/loopback paths this is -/// always null. -struct WorkItem { - uint32_t function_id; - std::vector frame_buf; ///< RPCHeader + payload (moved from RxFrame) - PeerId peer; ///< response destination - uint32_t request_id; ///< echoed from RPCHeader - uint64_t ptp_timestamp; - uint32_t vp_id; - ITransceiver *response_transport; ///< transport the request arrived on - ReleaseFn release_fn; ///< null except on GPU ring-buffer path -}; - -/// RAII wrapper: calls decoder::release_decode_graph() on destruction. -struct GraphResourcesDeleter { - cudaq::qec::decoder *owner = nullptr; - void operator()(void *p) const noexcept { - if (p && owner) - owner->release_decode_graph(p); - } -}; -using GraphResourcesPtr = std::unique_ptr; - -inline constexpr size_t kDefaultQueueDepth = 64; - -/// Owns one decoder instance plus a dedicated FIFO worker thread; decoder calls -/// are sequenced through the worker. -struct DecodingSession { - enum class ShotState { collecting, result_ready, failed }; - - // -- Decoder and GPU resources -- - std::unique_ptr dec; - GraphResourcesPtr graph_resources; - SyndromeMappingTable mapping_table; - - // -- Round assembly -- - RoundAccumulator accumulator; - - // -- Worker thread -- - std::thread worker; - std::queue work_queue; - std::mutex queue_mutex; - std::condition_variable queue_cv; - std::atomic shutdown{false}; - size_t queue_depth{kDefaultQueueDepth}; - - // Latched when enqueue_syndromes is dropped (queue full). The shot remains - // poisoned until reset_decoder clears all mutable state. - std::atomic syndromes_dropped{false}; - - // Worker-owned state for the current shot. result_ready means a decode call - // completed; it is deliberately independent of decoder_result::converged. - ShotState shot_state = ShotState::collecting; - size_t accepted_syndromes = 0; - - // Per-session metrics (updated atomically by the worker thread). - std::atomic enqueue_count{0}; - std::atomic decode_count{0}; - std::atomic get_corrections_count{0}; - std::atomic reset_count{0}; - std::atomic error_count{0}; - std::atomic busy_count{0}; - std::atomic syndromes_dropped_count{0}; - - DecodingSession() = default; - DecodingSession(const DecodingSession &) = delete; - DecodingSession &operator=(const DecodingSession &) = delete; - DecodingSession(DecodingSession &&) = delete; - DecodingSession &operator=(DecodingSession &&) = delete; - ~DecodingSession(); - - /// Construct a session around an already configured decoder and capture graph - /// resources if supported. - static std::unique_ptr - create(std::unique_ptr decoder, - SyndromeMappingTable mapping_table); - - /// Start the FIFO worker thread. Must be called after create(). The - /// worker pins itself to the decoder's cuda_device_id before serving work; - /// a pin failure throws HERE (one worker owns one decoder -- a worker on - /// the wrong device must never serve). - void start_worker(); - - /// Signal shutdown and join the worker (drains any queued items first). - /// Idempotent; also called from the destructor. DecodingServer calls this - /// before its transports are destroyed because queued items reply through - /// raw ITransceiver pointers. - void stop_worker(); - - /// Non-blocking enqueue. Returns false if the work queue is full. - bool try_enqueue(WorkItem item); - - /// Latch the syndromes_dropped flag (called by dispatcher on queue-full - /// enqueue_syndromes; no response is sent to the client). - void latch_syndromes_dropped(); - - // -- Worker-thread-only methods -- - void on_enqueue(const WorkItem &item); - void on_get_corrections(const WorkItem &item); - void on_reset(const WorkItem &item); - void worker_loop(); -}; - -/// High-water mark of simultaneously-busy DecodingSession workers across all -/// sessions in this process (concurrency evidence for multi-logical-qubit -/// tests and server stats). -uint64_t max_concurrent_busy_sessions(); - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceFactory.cpp b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceFactory.cpp deleted file mode 100644 index 651290121..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceFactory.cpp +++ /dev/null @@ -1,27 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -// Strong definition of the GPU RoCE factory that DecodingServer.cpp declares -// weakly. This translation unit lives in cudaq-qec-decoding-server-gpuroce -// (NOT the core library) so that only binaries linking that component carry -// the DOCA / Hololink / CUDA-driver dependencies. Consumers must link the -// component WHOLE_ARCHIVE: the sole reference to this symbol is weak, which -// does not pull archive members on its own. - -#include "DecodingServer.h" // resolve_decode_device (core symbol) -#include "GpuRoceTransceiver.h" - -extern "C" cudaq::qec::decoding_server::ITransceiver * -cudaqx_qec_make_gpu_roce_transceiver(int pinned_cuda_device) { - using namespace cudaq::qec::decoding_server; - // The gpu_roce device is the decoder's cuda_device_id pin; resolve it here, - // inside the component, where GpuRoceConfig is visible. - auto cfg = GpuRoceConfig::from_env(); - cfg.gpu_id = resolve_decode_device(pinned_cuda_device); - return new GpuRoceTransceiver(cfg); -} diff --git a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceLinkCheck.cpp b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceLinkCheck.cpp deleted file mode 100644 index 9fc37fe36..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceLinkCheck.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -// Link canary for the GPU RoCE component -- not meant to be executed -// (running it would require HOLOLINK_* env, a GPU driver, and RDMA-capable -// hardware). Building it forces the linker to resolve GpuRoceTransceiver's -// full dependency chain (hololink, DOCA, CUDA driver stubs), so HSB API -// drift is caught at build time even on machines where nothing links the -// component into a runnable binary (driverless CI: the decoding_server -// tool's gpu_roce block is additionally gated on the proprietary cudevice -// archive, which CI does not provision). - -namespace cudaq::qec::decoding_server { -struct ITransceiver; -} - -extern "C" cudaq::qec::decoding_server::ITransceiver * -cudaqx_qec_make_gpu_roce_transceiver(int pinned_cuda_device); - -using GpuRoceFactoryFn = cudaq::qec::decoding_server::ITransceiver *(*)(int); - -static GpuRoceFactoryFn volatile gpu_roce_factory = - &cudaqx_qec_make_gpu_roce_transceiver; - -int main() { return gpu_roce_factory ? 0 : 1; } diff --git a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp deleted file mode 100644 index df6ae4fba..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp +++ /dev/null @@ -1,466 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#ifdef CUDAQ_GPU_ROCE_AVAILABLE - -#include "GpuRoceTransceiver.h" -#include "RpcWireFormat.h" -#include "cudaq/qec/logger.h" -#include "cudaq/qec/realtime/graph_resources.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// CUDAQ device-graph scheduler API (cudaq-realtime-dispatch). -#include "cudaq/realtime/hololink_bridge_common.h" - -namespace cudaq::qec::decoding_server { - -// --------------------------------------------------------------------------- -// Internal helpers (same pattern as hololink_qldpc_graph_decoder_bridge.cpp) -// --------------------------------------------------------------------------- - -namespace { - -// Allocate \p bytes of CUDA pinned+mapped host memory and return both the host -// pointer and its device-mapped counterpart. The memory is zero-initialised. -bool alloc_pinned_mapped(size_t bytes, void **host_out, void **dev_out) { - void *h = nullptr; - if (cudaHostAlloc(&h, bytes, cudaHostAllocMapped) != cudaSuccess) - return false; - void *d = nullptr; - if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) { - cudaFreeHost(h); - return false; - } - std::memset(h, 0, bytes); - *host_out = h; - *dev_out = d; - return true; -} - -// Resolve a proprietary DEVICE_CALL populate shim via dlsym and stamp the -// function table entry. The server process must absorb -// libcudaq-qec-realtime-cudevice-proprietary.a (WHOLE_ARCHIVE) and link with -// --export-dynamic so the symbols are visible. -using populate_fn = void (*)(void *); -bool populate_device_call(cudaq_function_entry_t &entry, const char *symbol, - uint32_t function_id) { - auto fn = reinterpret_cast(::dlsym(RTLD_DEFAULT, symbol)); - if (!fn) { - CUDA_QEC_ERROR( - "GpuRoceTransceiver: dlsym({}) failed -- the server process must " - "absorb libcudaq-qec-realtime-cudevice-proprietary.a as WHOLE_ARCHIVE " - "and link with --export-dynamic", - symbol); - return false; - } - fn(&entry); - entry.function_id = function_id; - entry.routing_key = 0; - if (entry.dispatch_mode != CUDAQ_DISPATCH_DEVICE_CALL || - !entry.handler.device_fn_ptr) { - CUDA_QEC_ERROR("GpuRoceTransceiver: {} did not produce a valid " - "DEVICE_CALL entry", - symbol); - return false; - } - return true; -} - -#define GPU_CUDA_CHECK(expr) \ - do { \ - cudaError_t _err = (expr); \ - if (_err != cudaSuccess) \ - throw std::runtime_error( \ - std::string("GpuRoceTransceiver CUDA error: ") + \ - cudaGetErrorString(_err) + " (" #expr ")"); \ - } while (0) - -} // namespace - -// --------------------------------------------------------------------------- -// GpuRoceConfig::from_env -// --------------------------------------------------------------------------- - -static std::string env_str(const char *name, const char *def = "") { - const char *v = std::getenv(name); - return v ? v : def; -} -static uint32_t env_u32(const char *name, uint32_t def) { - const char *v = std::getenv(name); - return v ? static_cast(std::stoul(v)) : def; -} -static int env_int(const char *name, int def) { - const char *v = std::getenv(name); - return v ? std::stoi(v) : def; -} -static size_t env_size(const char *name, size_t def) { - const char *v = std::getenv(name); - return v ? static_cast(std::stoull(v)) : def; -} - -GpuRoceConfig GpuRoceConfig::from_env() { - GpuRoceConfig c; - c.device_name = env_str("HOLOLINK_DEVICE"); - c.peer_ip = env_str("HOLOLINK_PEER_IP"); - c.remote_qp = env_u32("HOLOLINK_REMOTE_QP", 0); - // gpu_id is not read from the environment: the device is the decoder's - // cuda_device_id, resolved by resolve_decode_device() at transport - // creation. - c.frame_size = env_size("HOLOLINK_FRAME_SIZE", 384); - c.page_size = env_size("HOLOLINK_PAGE_SIZE", 0); // 0 → derived below - c.num_pages = env_size("HOLOLINK_NUM_PAGES", 64); - c.reserved_sms = env_int("HOLOLINK_RESERVED_SMS", 2); - return c; -} - -// --------------------------------------------------------------------------- -// GpuRoceTransceiver constructor -// --------------------------------------------------------------------------- - -GpuRoceTransceiver::GpuRoceTransceiver(const GpuRoceConfig &config) - : gpu_id_(config.gpu_id) { - if (config.device_name.empty()) - throw std::runtime_error("GpuRoceTransceiver: HOLOLINK_DEVICE not set"); - if (config.peer_ip.empty()) - throw std::runtime_error("GpuRoceTransceiver: HOLOLINK_PEER_IP not set"); - if (config.remote_qp == 0) - throw std::runtime_error("GpuRoceTransceiver: HOLOLINK_REMOTE_QP not set"); - - // Derive page_size from frame_size if not overridden, then round up to the - // 128-byte Hololink granularity. Mirrors the derivation in - // hololink_qldpc_graph_decoder_bridge.cpp (lines 279-282). - size_t page_size = config.page_size ? config.page_size : config.frame_size; - page_size = (page_size + 127) & ~static_cast(127); - - if (page_size != 0 && - config.num_pages > std::numeric_limits::max() / page_size) - throw std::runtime_error( - "GpuRoceTransceiver: ring size overflow for " - "HOLOLINK_FRAME_SIZE/HOLOLINK_PAGE_SIZE=" + - std::to_string(page_size) + - " and HOLOLINK_NUM_PAGES=" + std::to_string(config.num_pages)); - const size_t ring_bytes = page_size * config.num_pages; - const long host_page_size = ::sysconf(_SC_PAGESIZE); - if (host_page_size > 0 && - ring_bytes % static_cast(host_page_size) != 0) - throw std::runtime_error( - "GpuRoceTransceiver: ring buffer size " + std::to_string(ring_bytes) + - " bytes is not aligned to host page size " + - std::to_string(host_page_size) + - " bytes; adjust HOLOLINK_NUM_PAGES or HOLOLINK_PAGE_SIZE"); - - // Matches the call shape in hololink_qldpc_graph_decoder_bridge.cpp (lines - // 288-291). - transceiver_ = hololink_create_transceiver( - config.device_name.c_str(), - /*arg1=*/1, config.remote_qp, config.gpu_id, config.frame_size, page_size, - config.num_pages, config.peer_ip.c_str(), - /*forward=*/0, - /*rx_only=*/1, - /*tx_only=*/1); - if (!transceiver_) - throw std::runtime_error( - "GpuRoceTransceiver: hololink_create_transceiver() failed for device=" + - config.device_name + " peer=" + config.peer_ip); - - // Do NOT destroy a half-initialized transceiver on start failure (mirrors - // the guard at lines 297-306 in the bridge: DOCA teardown may double-free - // GPU memory that was never allocated, causing a segfault). - if (!hololink_start(transceiver_)) - throw std::runtime_error( - "GpuRoceTransceiver: hololink_start() failed (check that the IB " - "netdev has an IPv4 address assigned for RoCE v2 GID)"); - - // Adopt the DOCA ring buffer GPU VRAM pointers. - rx_ring_data_ = - reinterpret_cast(hololink_get_rx_ring_data_addr(transceiver_)); - rx_ring_flag_ = reinterpret_cast( - hololink_get_rx_ring_flag_addr(transceiver_)); - tx_ring_data_ = - reinterpret_cast(hololink_get_tx_ring_data_addr(transceiver_)); - tx_ring_flag_ = reinterpret_cast( - hololink_get_tx_ring_flag_addr(transceiver_)); - - if (!rx_ring_data_ || !rx_ring_flag_ || !tx_ring_data_ || !tx_ring_flag_) { - hololink_close(transceiver_); - hololink_destroy_transceiver(transceiver_); - transceiver_ = {}; - throw std::runtime_error( - "GpuRoceTransceiver: null DOCA ring pointer(s) after hololink_start"); - } - - num_pages_ = hololink_get_num_pages(transceiver_); - page_size_ = hololink_get_page_size(transceiver_); - - CUDA_QEC_INFO("GpuRoceTransceiver: Hololink started device={} peer={} " - "gpu={} pages={} page_size={} " - "QP=0x{:X} rkey={} buf=0x{:X} " - "(call launch_scheduler() before run())", - config.device_name, config.peer_ip, config.gpu_id, num_pages_, - page_size_, hololink_get_qp_number(transceiver_), - hololink_get_rkey(transceiver_), - hololink_get_buffer_addr(transceiver_)); -} - -// --------------------------------------------------------------------------- -// launch_scheduler -// --------------------------------------------------------------------------- - -void GpuRoceTransceiver::launch_scheduler(void *raw_graph_resources) { - auto *graph_res = - static_cast(raw_graph_resources); - if (!graph_res || !graph_res->graph_exec) - throw std::runtime_error( - "GpuRoceTransceiver::launch_scheduler: null graph_exec " - "(decoder must support_graph_dispatch() and capture_decode_graph())"); - - GPU_CUDA_CHECK(cudaSetDevice(gpu_id_)); - - void *ft_dev = nullptr; - if (!alloc_pinned_mapped(3 * sizeof(cudaq_function_entry_t), &ft_host_, - &ft_dev)) - throw std::runtime_error("GpuRoceTransceiver::launch_scheduler: " - "function-table pinned alloc failed"); - - auto *entries = static_cast(ft_host_); - bool ok = - populate_device_call(entries[0], - "cudaqx_qec_realtime_dispatch_populate_enqueue_" - "syndromes_device_entry", - kEnqueueSyndromesFunctionId) && - populate_device_call( - entries[1], - "cudaqx_qec_realtime_dispatch_populate_get_corrections_device_entry", - kGetCorrectionsFunctionId) && - populate_device_call( - entries[2], - "cudaqx_qec_realtime_dispatch_populate_reset_decoder_device_entry", - kResetDecoderFunctionId); - if (!ok) { - cudaFreeHost(ft_host_); - ft_host_ = nullptr; - throw std::runtime_error( - "GpuRoceTransceiver::launch_scheduler: populate_device_call failed " - "(see error log above)"); - } - - // Resolve dispatch graph API via dlsym; cudaq-realtime-dispatch is linked - // into the server (not this static lib) to keep the CUDA module in one copy. - // Signatures must match create/launch/destroy_dispatch_graph_fn_t in - // qec_realtime_session.cpp/.h exactly — calling-convention mismatch is UB. - using create_fn_t = cudaError_t (*)( - volatile std::uint64_t *, volatile std::uint64_t *, std::uint8_t *, - std::uint8_t *, std::size_t, std::size_t, cudaq_function_entry_t *, - std::size_t, void *, volatile int *, std::uint64_t *, std::size_t, - std::uint32_t, std::uint32_t, cudaGraphExec_t, cudaStream_t, - cudaq_dispatch_graph_context **); - using launch_fn_t = - cudaError_t (*)(cudaq_dispatch_graph_context *, cudaStream_t); - using destroy_fn_t = cudaError_t (*)(cudaq_dispatch_graph_context *); - - auto create_dispatch = reinterpret_cast( - ::dlsym(RTLD_DEFAULT, "cudaq_create_dispatch_graph_regular")); - auto launch_dispatch = reinterpret_cast( - ::dlsym(RTLD_DEFAULT, "cudaq_launch_dispatch_graph")); - auto destroy_dispatch = reinterpret_cast( - ::dlsym(RTLD_DEFAULT, "cudaq_destroy_dispatch_graph")); - - if (!create_dispatch || !launch_dispatch || !destroy_dispatch) { - cudaFreeHost(ft_host_); - ft_host_ = nullptr; - CUDA_QEC_ERROR( - "GpuRoceTransceiver: cudaq dispatch API not found via dlsym -- " - "the server must link cudaq-realtime-dispatch with --export-dynamic"); - throw std::runtime_error( - "GpuRoceTransceiver::launch_scheduler: cudaq dispatch API not found " - "(cudaq_create/launch/destroy_dispatch_graph_regular); " - "link cudaq-realtime-dispatch into the server with --export-dynamic"); - } - fn_destroy_dispatch_graph_ = destroy_dispatch; - - void *sd_host = nullptr, *sd_dev = nullptr; - if (!alloc_pinned_mapped(sizeof(int), &sd_host, &sd_dev)) { - cudaFreeHost(ft_host_); - ft_host_ = nullptr; - throw std::runtime_error("GpuRoceTransceiver::launch_scheduler: " - "shutdown-flag pinned alloc failed"); - } - shutdown_host_ = static_cast(sd_host); - shutdown_dev_ = static_cast(sd_dev); - - if (cudaMalloc(&d_stats_, sizeof(uint64_t)) != cudaSuccess || - cudaMemset(d_stats_, 0, sizeof(uint64_t)) != cudaSuccess) { - cudaFreeHost(ft_host_); - ft_host_ = nullptr; - cudaFreeHost(sd_host); - shutdown_host_ = nullptr; - shutdown_dev_ = nullptr; - throw std::runtime_error( - "GpuRoceTransceiver::launch_scheduler: d_stats_ alloc failed"); - } - - GPU_CUDA_CHECK(cudaStreamCreate(&sched_stream_)); - - cudaError_t cerr = create_dispatch( - rx_ring_flag_, tx_ring_flag_, rx_ring_data_, tx_ring_data_, page_size_, - page_size_, static_cast(ft_dev), - /*func_count=*/3, - /*graph_io_ctx=*/nullptr, shutdown_dev_, d_stats_, num_pages_, - /*num_blocks=*/1, /*threads_per_block=*/64, graph_res->graph_exec, - sched_stream_, &sched_ctx_); - if (cerr != cudaSuccess) { - cudaStreamDestroy(sched_stream_); - sched_stream_ = nullptr; - cudaFree(d_stats_); - d_stats_ = nullptr; - cudaFreeHost(ft_host_); - ft_host_ = nullptr; - cudaFreeHost(sd_host); - shutdown_host_ = nullptr; - shutdown_dev_ = nullptr; - throw std::runtime_error( - std::string("GpuRoceTransceiver::launch_scheduler: " - "cudaq_create_dispatch_graph_regular: ") + - cudaGetErrorString(cerr)); - } - - cerr = launch_dispatch(sched_ctx_, sched_stream_); - if (cerr != cudaSuccess) { - fn_destroy_dispatch_graph_(sched_ctx_); - sched_ctx_ = nullptr; - cudaStreamDestroy(sched_stream_); - sched_stream_ = nullptr; - cudaFree(d_stats_); - d_stats_ = nullptr; - cudaFreeHost(ft_host_); - ft_host_ = nullptr; - cudaFreeHost(sd_host); - shutdown_host_ = nullptr; - shutdown_dev_ = nullptr; - throw std::runtime_error( - std::string("GpuRoceTransceiver::launch_scheduler: " - "cudaq_launch_dispatch_graph: ") + - cudaGetErrorString(cerr)); - } - - monitor_thread_ = - std::thread([this] { hololink_blocking_monitor(transceiver_); }); - - CUDA_QEC_INFO("GpuRoceTransceiver: GPU scheduler launched " - "QP=0x{:X} rkey={} buf=0x{:X} " - "(3 DEVICE_CALL entries, graph_exec={:p})", - hololink_get_qp_number(transceiver_), - hololink_get_rkey(transceiver_), - hololink_get_buffer_addr(transceiver_), - static_cast(graph_res->graph_exec)); - - // Print RDMA target info to stdout so the orchestration script can grep it. - // Matches the format in hololink_qldpc_graph_decoder_bridge.cpp lines - // 441-444. - std::cout << "QP Number: 0x" << std::hex - << hololink_get_qp_number(transceiver_) << std::dec << "\n" - << "RKey: " << hololink_get_rkey(transceiver_) << "\n" - << "Buffer Addr: 0x" << std::hex - << hololink_get_buffer_addr(transceiver_) << std::dec << "\n"; - std::cout.flush(); -} - -// --------------------------------------------------------------------------- -// ITransceiver interface stubs (GPU scheduler handles the data path) -// --------------------------------------------------------------------------- - -RxFrame GpuRoceTransceiver::recv() { - // The GPU device-graph scheduler handles RX→dispatch→decode→TX autonomously. - // This method only exists so DecodingServer::run()'s recv loop blocks until - // shutdown() is called. - while (!stopped_.load(std::memory_order_acquire)) - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - return {}; // shutdown sentinel: empty buf causes the recv loop to exit -} - -void GpuRoceTransceiver::send(const PeerId & /*peer*/, const uint8_t * /*data*/, - size_t /*len*/) { - throw std::logic_error( - "GpuRoceTransceiver::send() must not be called: the CUDAQ device-graph " - "scheduler writes TX responses directly to the Hololink ring buffer"); -} - -// --------------------------------------------------------------------------- -// shutdown / destructor -// --------------------------------------------------------------------------- - -void GpuRoceTransceiver::shutdown() { - if (stopped_.exchange(true, std::memory_order_acq_rel)) - return; // already stopped - - // Signal the GPU scheduler kernel to stop its self-relaunch loop. - if (shutdown_host_) - __atomic_store_n(shutdown_host_, 1, __ATOMIC_RELEASE); - - // Stop the Hololink RX/TX kernels to unblock hololink_blocking_monitor(). - if (transceiver_) - hololink_close(transceiver_); -} - -GpuRoceTransceiver::~GpuRoceTransceiver() { - // Ensure clean shutdown even if the caller omitted shutdown(). - if (!stopped_.exchange(true, std::memory_order_acq_rel)) { - if (shutdown_host_) - __atomic_store_n(shutdown_host_, 1, __ATOMIC_RELEASE); - if (transceiver_) - hololink_close(transceiver_); - } - - if (monitor_thread_.joinable()) - monitor_thread_.join(); - - if (sched_stream_) { - cudaStreamSynchronize(sched_stream_); // drain the self-relaunch chain - if (sched_ctx_ && fn_destroy_dispatch_graph_) - fn_destroy_dispatch_graph_(sched_ctx_); - cudaStreamDestroy(sched_stream_); - } - - if (ft_host_) - cudaFreeHost(ft_host_); - if (shutdown_host_) - cudaFreeHost(const_cast(shutdown_host_)); - if (d_stats_) - cudaFree(d_stats_); - if (transceiver_) - hololink_destroy_transceiver(transceiver_); -} - -// --------------------------------------------------------------------------- -// RDMA target info -// --------------------------------------------------------------------------- - -uint32_t GpuRoceTransceiver::qp_number() const { - return hololink_get_qp_number(transceiver_); -} -uint32_t GpuRoceTransceiver::rkey() const { - return hololink_get_rkey(transceiver_); -} -uint64_t GpuRoceTransceiver::buffer_addr() const { - return hololink_get_buffer_addr(transceiver_); -} - -} // namespace cudaq::qec::decoding_server - -#endif // CUDAQ_GPU_ROCE_AVAILABLE diff --git a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.h b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.h deleted file mode 100644 index 89200f81a..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.h +++ /dev/null @@ -1,137 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#ifdef CUDAQ_GPU_ROCE_AVAILABLE - -#include "ITransceiver.h" - -#include -#include -#include -#include -#include - -#include - -// Hololink Sensor Bridge + CUDAQ dispatcher C API. -// Both are provided by cudaq-realtime-bridge-hololink and -// cudaq-realtime-dispatch. -#include "cudaq/realtime/daemon/bridge/hololink/hololink_wrapper.h" - -// Forward-declare the opaque scheduler context so the header stays independent -// of the full cudaq_realtime.h. -struct cudaq_dispatch_graph_context; - -namespace cudaq::qec::decoding_server { - -/// Runtime configuration for GpuRoceTransceiver. Transport fields are read -/// from environment variables so that the server can be reconfigured without a -/// rebuild; gpu_id is the exception -- it is the decoder's cuda_device_id, -/// filled in at transport creation. -struct GpuRoceConfig { - std::string device_name; ///< HOLOLINK_DEVICE (IB netdev, e.g. "mlx5_0") - uint32_t remote_qp{0}; ///< HOLOLINK_REMOTE_QP (FPGA/emulator QP number) - int gpu_id{0}; ///< FPGA-affine GPU; set from the decoder's - ///< cuda_device_id by resolve_decode_device() - size_t frame_size{384}; ///< HOLOLINK_FRAME_SIZE (max RPC frame bytes) - size_t page_size{0}; ///< HOLOLINK_PAGE_SIZE (0 → derived from frame_size) - size_t num_pages{64}; ///< HOLOLINK_NUM_PAGES (ring depth) - std::string peer_ip; ///< HOLOLINK_PEER_IP (FPGA/emulator IPv4) - int reserved_sms{2}; ///< HOLOLINK_RESERVED_SMS (SMs for Hololink RX/TX) - - static GpuRoceConfig from_env(); -}; - -/// GPU RoCE transport and device-graph scheduler for the decoding server. -/// -/// ## Architecture -/// -/// Hololink DMA's RPC frames from the FPGA directly into DOCA GPU ring buffers. -/// `launch_scheduler()` wires those ring buffers to the CUDAQ device-graph -/// scheduler (`cudaq_create_dispatch_graph_regular`) and the captured decoder -/// CUDA graph, replicating the pattern in -/// `libs/qec/unittests/utils/hololink_qldpc_graph_decoder_bridge.cpp`. -/// -/// After `launch_scheduler()` returns, the GPU handles the full -/// RX → dispatch → decode → TX loop autonomously. No CPU `recv()` or `send()` -/// is involved in the data path; those methods are stubs that satisfy the -/// `ITransceiver` contract used by `DecodingServer::run()`. -/// -/// ## Multi-decoder -/// -/// Currently limited to a single decoder session (enforced by DecodingServer). -/// Multi-decoder GPU RoCE with per-session ring binding is deferred. -class GpuRoceTransceiver final : public ITransceiver { -public: - explicit GpuRoceTransceiver(const GpuRoceConfig &config); - ~GpuRoceTransceiver() override; - - /// Wire the DOCA ring buffers to the CUDAQ device-graph scheduler and launch - /// the GPU dispatch loop. Must be called exactly once after the transceiver - /// is created and before `run()`. - /// - /// \p raw_graph_resources is the `void *` returned by - /// `decoder::capture_decode_graph()`; it is cast internally to - /// `cudaq::qec::realtime::graph_resources *` to extract `graph_exec`. - void launch_scheduler(void *raw_graph_resources); - - /// ITransceiver hook: forwards to launch_scheduler(). - bool launch_device_scheduler(void *raw_graph_resources) override { - launch_scheduler(raw_graph_resources); - return true; - } - - /// Block until shutdown() is called. The GPU scheduler handles RX/TX; - /// this method only satisfies the ITransceiver contract for DecodingServer. - RxFrame recv() override; - - /// Not used on the GPU scheduler path — the device graph kernel writes TX - /// responses directly. Always throws std::logic_error. - void send(const PeerId &peer, const uint8_t *data, size_t len) override; - - void shutdown() override; - - /// RDMA target info printed after launch_scheduler() for the orchestration - /// script (QP number, rkey, buffer address). - uint32_t qp_number() const; - uint32_t rkey() const; - uint64_t buffer_addr() const; - -private: - hololink_transceiver_t transceiver_{}; - int gpu_id_{0}; - - // DOCA ring buffer pointers (GPU VRAM — device addresses). - uint8_t *rx_ring_data_{nullptr}; - volatile uint64_t *rx_ring_flag_{nullptr}; - uint8_t *tx_ring_data_{nullptr}; - volatile uint64_t *tx_ring_flag_{nullptr}; - size_t num_pages_{0}; - size_t page_size_{0}; - - // CUDAQ device-graph scheduler state (set by launch_scheduler). - cudaq_dispatch_graph_context *sched_ctx_{nullptr}; - cudaStream_t sched_stream_{nullptr}; - void *ft_host_{nullptr}; ///< pinned host ptr: function entry table - volatile int *shutdown_host_{ - nullptr}; ///< pinned host ptr: GPU scheduler stop flag - volatile int *shutdown_dev_{nullptr}; ///< device-mapped ptr of shutdown_host_ - uint64_t *d_stats_{nullptr}; - // Cached from launch_scheduler() so the destructor can call it without dlsym. - cudaError_t (*fn_destroy_dispatch_graph_)(cudaq_dispatch_graph_context *){ - nullptr}; - - std::atomic stopped_{false}; - std::thread monitor_thread_; ///< runs hololink_blocking_monitor() -}; - -} // namespace cudaq::qec::decoding_server - -#endif // CUDAQ_GPU_ROCE_AVAILABLE diff --git a/libs/qec/lib/realtime/decoding-server-cqr/ITransceiver.h b/libs/qec/lib/realtime/decoding-server-cqr/ITransceiver.h deleted file mode 100644 index 6e2c7270f..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/ITransceiver.h +++ /dev/null @@ -1,105 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace cudaq::qec::decoding_server { - -/// Peer identity — the address to which the server sends a response. -struct PeerId { - std::array addr; ///< GID / IPv6 (16 bytes) - uint16_t port; - - bool operator==(const PeerId &) const = default; -}; - -/// Move-only wrapper that invokes the wrapped callback exactly once, when the -/// holder is destroyed. Frames are dropped on many paths (dispatcher -/// validation failures, session queue full, handler exceptions); tying the -/// release to the frame's lifetime guarantees the ring slot is returned on -/// every one of them instead of relying on each path to call it by hand. -class ReleaseFn { -public: - ReleaseFn() = default; - explicit ReleaseFn(std::function fn) : fn_(std::move(fn)) {} - ReleaseFn(const ReleaseFn &) = delete; - ReleaseFn &operator=(const ReleaseFn &) = delete; - ReleaseFn(ReleaseFn &&other) noexcept - : fn_(std::exchange(other.fn_, nullptr)) {} - ReleaseFn &operator=(ReleaseFn &&other) noexcept { - if (this != &other) { - if (fn_) - fn_(); - fn_ = std::exchange(other.fn_, nullptr); - } - return *this; - } - ~ReleaseFn() { - if (fn_) - fn_(); - } - explicit operator bool() const noexcept { return static_cast(fn_); } - -private: - std::function fn_; -}; - -/// A received frame: owns the wire bytes (RPCHeader + payload) plus transport -/// metadata needed for syndrome scatter and response routing. -/// Ownership of buf is transferred to WorkItem on enqueue. -/// -/// release_fn: when non-null, returns the transport ring slot; it fires -/// automatically when the frame (or the WorkItem it was moved into) is -/// destroyed. For host-copy transports (CQR, Loopback, CPU ring buffer path) -/// it is always null — the copy itself constitutes "release." For GPU ring -/// buffer transports (full RelayBP path), the frame must be kept alive until -/// GPU decode completes so the slot is not returned to Hololink early. -struct RxFrame { - std::vector buf; ///< RPCHeader + payload (owned copy) - uint32_t vp_id = 0; - PeerId peer{}; - ReleaseFn release_fn; ///< null except on GPU ring-buffer path -}; - -/// Transport abstraction used by DecodingServer and DecodingSession. -struct ITransceiver { - /// Block until a frame is available and return it (buf is owned by caller). - /// After shutdown() this may return a frame with an EMPTY buf -- the - /// sentinel that unblocks the receive loop so it can observe the shutdown - /// flag and exit. - virtual RxFrame recv() = 0; - - /// Unblock any thread waiting in recv() (which then returns an empty - /// sentinel frame). Called by DecodingServer::stop(). - virtual void shutdown() {} - - /// Send a response to \p peer. Thread-safe: called from session worker - /// threads, which may be concurrent. - virtual void send(const PeerId &peer, const uint8_t *data, size_t len) = 0; - - /// Optional hook for GPU ring-buffer transports: wire \p graph_resources - /// (the opaque pointer from decoder::capture_decode_graph()) into an - /// on-device scheduler and launch it. Returns false when the transport has - /// no device scheduler (the default); callers that require one treat that - /// as an error. Keeps DecodingServer free of concrete-transceiver types - /// (and their link-time dependencies). - virtual bool launch_device_scheduler(void * /*graph_resources*/) { - return false; - } - - virtual ~ITransceiver() = default; -}; - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/LoopbackTransceiver.h b/libs/qec/lib/realtime/decoding-server-cqr/LoopbackTransceiver.h deleted file mode 100644 index 9a97ffcfd..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/LoopbackTransceiver.h +++ /dev/null @@ -1,102 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include "ITransceiver.h" - -#include -#include -#include -#include -#include - -namespace cudaq::qec::decoding_server { - -/// Paired in-process transceivers for development and testing. -/// -/// LoopbackTransceiver::make() returns two endpoints A and B. -/// Data written with A.send() is readable via B.recv(), and vice versa. -/// All payload bytes are copied on enqueue so each RxFrame owns its buffer. -class LoopbackTransceiver final : public ITransceiver { -public: - static std::pair, - std::unique_ptr> - make(); - - RxFrame recv() override; - - void send(const PeerId &peer, const uint8_t *data, size_t len) override; - - void shutdown() override; - -private: - explicit LoopbackTransceiver( - std::shared_ptr>> inbox, - std::shared_ptr>> outbox, - std::shared_ptr mtx, - std::shared_ptr cv, - std::shared_ptr stopped) - : inbox_(std::move(inbox)), outbox_(std::move(outbox)), - mtx_(std::move(mtx)), cv_(std::move(cv)), stopped_(std::move(stopped)) { - } - - std::shared_ptr>> inbox_; - std::shared_ptr>> outbox_; - std::shared_ptr mtx_; - std::shared_ptr cv_; - // Shared between the two endpoints so that shutting down either side wakes - // a peer blocked in recv() on the other. - std::shared_ptr stopped_; -}; - -inline std::pair, - std::unique_ptr> -LoopbackTransceiver::make() { - auto q_ab = std::make_shared>>(); - auto q_ba = std::make_shared>>(); - auto mtx = std::make_shared(); - auto cv = std::make_shared(); - auto stopped = std::make_shared(false); - - auto a = std::unique_ptr( - new LoopbackTransceiver(q_ba, q_ab, mtx, cv, stopped)); - auto b = std::unique_ptr( - new LoopbackTransceiver(q_ab, q_ba, mtx, cv, stopped)); - return {std::move(a), std::move(b)}; -} - -inline RxFrame LoopbackTransceiver::recv() { - std::unique_lock lk(*mtx_); - cv_->wait(lk, [this] { return !inbox_->empty() || *stopped_; }); - if (inbox_->empty()) - return {}; // shutdown sentinel (empty buf) - RxFrame frame; - frame.buf = std::move(inbox_->front()); - inbox_->pop_front(); - return frame; -} - -inline void LoopbackTransceiver::shutdown() { - { - std::lock_guard lk(*mtx_); - *stopped_ = true; - } - cv_->notify_all(); -} - -inline void LoopbackTransceiver::send(const PeerId & /*peer*/, - const uint8_t *data, size_t len) { - { - std::lock_guard lk(*mtx_); - outbox_->emplace_back(data, data + len); - } - cv_->notify_all(); -} - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/RoundAccumulator.cpp b/libs/qec/lib/realtime/decoding-server-cqr/RoundAccumulator.cpp deleted file mode 100644 index 85e0798f8..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/RoundAccumulator.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#include "RoundAccumulator.h" - -#include -#include - -namespace cudaq::qec::decoding_server { - -std::size_t RoundKeyHash::operator()(const RoundKey &k) const noexcept { - // FNV-1a style mix over the three fields. - std::size_t h = 14695981039346656037ULL; - auto mix = [&](uint64_t v) { - h ^= v; - h *= 1099511628211ULL; - }; - mix(k.decoder_id); - mix(k.counter); - mix(k.syndrome_mapping_id); - return h; -} - -std::optional -RoundAccumulator::ingest(const RoundKey &key, uint32_t vp_id, - const uint8_t *bits, size_t num_syndromes, - const SyndromeMappingTable &table) { - auto table_it = table.find(key.syndrome_mapping_id); - if (table_it == table.end()) - throw std::invalid_argument("Unknown syndrome_mapping_id: " + - std::to_string(key.syndrome_mapping_id)); - - const auto &vp_mappings = table_it->second; - - if (vp_id >= static_cast(vp_mappings.size())) - throw std::invalid_argument("VP " + std::to_string(vp_id) + - " not in syndrome mapping row"); - - const auto &indices = vp_mappings[vp_id]; - const bool pass_through = indices.empty(); - - if (pass_through && vp_mappings.size() != 1) - throw std::invalid_argument( - "Pass-through syndrome mapping requires exactly one VP"); - - if (!pass_through && num_syndromes != indices.size()) - throw std::invalid_argument("Syndrome count mismatch: got " + - std::to_string(num_syndromes) + " expected " + - std::to_string(indices.size())); - - auto &round = rounds_[key]; - if (round.flat.empty()) { - if (pass_through) { - // Pass-through: flat vector sized to this VP's contribution. - round.flat.assign(num_syndromes, 0); - } else { - // Index-mapped: flat vector sized to the union of all VP index ranges. - uint32_t max_idx = 0; - for (const auto &vp_map : vp_mappings) - for (uint32_t idx : vp_map) - if (idx > max_idx) - max_idx = idx; - round.flat.assign(max_idx + 1, 0); - } - round.expected_vp_count = static_cast(vp_mappings.size()); - round.type = RoundType::BULK; - } - - if (round.received_vps.count(vp_id)) - throw std::invalid_argument("Duplicate VP fragment for vp_id=" + - std::to_string(vp_id)); - - if (pass_through) { - for (size_t i = 0; i < num_syndromes; ++i) - round.flat[i] = bits[i]; - } else { - for (size_t i = 0; i < num_syndromes; ++i) - round.flat[indices[i]] = bits[i]; - } - - round.received_vps.insert(vp_id); - - if (round.received_vps.size() < round.expected_vp_count) - return std::nullopt; - - CompletedRound result{ - .counter = key.counter, - .syndrome_mapping_id = key.syndrome_mapping_id, - .type = round.type, - .bits = std::move(round.flat), - }; - rounds_.erase(key); - return result; -} - -void RoundAccumulator::clear() { rounds_.clear(); } - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/RoundAccumulator.h b/libs/qec/lib/realtime/decoding-server-cqr/RoundAccumulator.h deleted file mode 100644 index f7f87de62..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/RoundAccumulator.h +++ /dev/null @@ -1,89 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace cudaq::qec::decoding_server { - -/// Round type forwarded from the syndrome mapping table. -enum class RoundType : uint8_t { - BULK = 0, ///< Mid-circuit syndrome-extraction round - FINAL = 1, ///< Last round (data-qubit measurements) -}; - -/// Completed round emitted by RoundAccumulator once all VP fragments arrive. -struct CompletedRound { - uint64_t counter; - uint64_t syndrome_mapping_id; - RoundType type; - std::vector - bits; ///< round-local flat detector vector (byte-per-bit) -}; - -/// Syndrome mapping table: -/// table[syndrome_mapping_id][vp_id] = vector of flat target indices -/// -/// For the 0.7.0 MVP (single VP), the table has one entry per -/// syndrome_mapping_id with one VP whose EMPTY index list denotes the -/// identity mapping: the fragment passes through as the completed round at -/// whatever length it arrives (the decoder accumulates rounds itself). -/// Non-empty index lists scatter round-relative indices per the spec's -/// syndrome mapping table. -using SyndromeMappingTable = - std::unordered_map>>; - -/// Key that identifies an in-progress round. -struct RoundKey { - uint64_t decoder_id; - uint64_t counter; - uint64_t syndrome_mapping_id; - bool operator==(const RoundKey &) const = default; -}; - -struct RoundKeyHash { - std::size_t operator()(const RoundKey &k) const noexcept; -}; - -/// Assembles VP syndrome fragments into the flat detector vector required by -/// the decoder. -/// -/// Owned by DecodingSession and accessed exclusively by that session's FIFO -/// worker thread. No internal locking is required. -class RoundAccumulator { -public: - /// Scatter bits from \p vp_id into the round buffer; returns CompletedRound - /// once all VPs contribute, nullopt otherwise. - /// @throws std::invalid_argument on unknown mapping id, duplicate VP, or - /// length mismatch. - std::optional ingest(const RoundKey &key, uint32_t vp_id, - const uint8_t *bits, - size_t num_syndromes, - const SyndromeMappingTable &table); - - /// Discard all in-progress rounds. Called by DecodingSession::on_reset(). - void clear(); - -private: - struct InProgressRound { - std::vector flat; - std::set received_vps; - uint32_t expected_vp_count = 0; - RoundType type = RoundType::BULK; - }; - - std::unordered_map rounds_; -}; - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/RpcDispatcher.cpp b/libs/qec/lib/realtime/decoding-server-cqr/RpcDispatcher.cpp deleted file mode 100644 index 8dd31f8c0..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/RpcDispatcher.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#include "RpcDispatcher.h" - -#include "cudaq/qec/logger.h" - -#include -#include - -namespace cudaq::qec::decoding_server { - -// --------------------------------------------------------------------------- -// ResponseWriter -// --------------------------------------------------------------------------- - -void ResponseWriter::write_error(RpcStatus status) { - std::vector buf(sizeof(RPCResponse)); - auto *hdr = reinterpret_cast(buf.data()); - hdr->magic = kRPCResponseMagic; - hdr->status = static_cast(status); - hdr->result_len = 0; - hdr->request_id = request_id_; - hdr->ptp_timestamp = ptp_timestamp_; - transport_.send(peer_, buf.data(), buf.size()); -} - -// --------------------------------------------------------------------------- -// RpcDispatcher -// --------------------------------------------------------------------------- - -void RpcDispatcher::register_handler(uint32_t function_id, Handler h) { - table_.emplace(function_id, std::move(h)); -} - -void RpcDispatcher::dispatch(RxFrame frame, ITransceiver &transport) { - // Minimum frame: RPCHeader only. - if (frame.buf.size() < sizeof(RPCHeader)) { - CUDA_QEC_DBG("RpcDispatcher: frame too short ({} bytes)", frame.buf.size()); - // Cannot build a meaningful response without a valid request_id. - return; - } - - const auto *hdr = reinterpret_cast(frame.buf.data()); - - if (hdr->magic != kRPCRequestMagic) { - CUDA_QEC_DBG("RpcDispatcher: bad magic 0x{:08X}", hdr->magic); - return; - } - - ResponseWriter writer(transport, frame.peer, hdr->request_id, - hdr->ptp_timestamp); - - auto it = table_.find(hdr->function_id); - if (it == table_.end()) { - CUDA_QEC_DBG("RpcDispatcher: unknown function_id 0x{:08X}", - hdr->function_id); - writer.write_error(RpcStatus::BAD_REQUEST); - return; - } - - try { - it->second(std::move(frame), writer); - } catch (const std::out_of_range &) { - // SessionRegistry::get() throws std::out_of_range for unknown decoder_id. - writer.write_error(RpcStatus::INVALID_DECODER); - } catch (const std::invalid_argument &) { - writer.write_error(RpcStatus::BAD_REQUEST); - } catch (const std::exception &e) { - cudaq::qec::error("RpcDispatcher: handler threw: {}", e.what()); - writer.write_error(RpcStatus::INTERNAL_ERROR); - } -} - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/RpcDispatcher.h b/libs/qec/lib/realtime/decoding-server-cqr/RpcDispatcher.h deleted file mode 100644 index cfa77fdf0..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/RpcDispatcher.h +++ /dev/null @@ -1,64 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include "ITransceiver.h" -#include "RpcWireFormat.h" - -#include -#include -#include - -namespace cudaq::qec::decoding_server { - -/// Helper passed to RpcDispatcher handlers that writes synchronous error -/// responses (e.g. BAD_REQUEST from header validation, BUSY from try_enqueue). -/// Successful responses are sent asynchronously by the session worker thread. -class ResponseWriter { -public: - ResponseWriter(ITransceiver &transport, const PeerId &peer, - uint32_t request_id, uint64_t ptp_timestamp) - : transport_(transport), peer_(peer), request_id_(request_id), - ptp_timestamp_(ptp_timestamp) {} - - void write_error(RpcStatus status); - - /// Expose the underlying transport so handlers can store it in WorkItem - /// for async responses from the worker thread. - ITransceiver *transport() const { return &transport_; } - -private: - ITransceiver &transport_; - PeerId peer_; - uint32_t request_id_; - uint64_t ptp_timestamp_; -}; - -/// Routes incoming RxFrames to registered handlers by function_id. -/// -/// Never calls decoder methods directly. Validates the frame header and -/// dispatches to the appropriate handler; on error, writes the error response -/// via ResponseWriter before returning. -class RpcDispatcher { -public: - /// Handler: enqueue a WorkItem or call writer.write_error() for sync errors. - /// RxFrame is passed by value so the handler can move buf into WorkItem. - using Handler = std::function; - - void register_handler(uint32_t function_id, Handler h); - - /// Validate header, look up function_id, and invoke the registered handler. - /// Writes BAD_REQUEST if the header is malformed or function_id is unknown. - void dispatch(RxFrame frame, ITransceiver &transport); - -private: - std::unordered_map table_; -}; - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/RpcWireFormat.h b/libs/qec/lib/realtime/decoding-server-cqr/RpcWireFormat.h deleted file mode 100644 index a84905dc3..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/RpcWireFormat.h +++ /dev/null @@ -1,100 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include -#include - -namespace cudaq::qec::decoding_server { - -// Function IDs for the three decoder RPCs (FNV1a-32 of function names). -// Values match the static_asserts in decoder_rpc_ids.h; duplicated here to -// avoid a dependency on CUDAQ_REALTIME headers. -inline constexpr uint32_t kEnqueueSyndromesFunctionId = 0x7ED8BE82u; -inline constexpr uint32_t kGetCorrectionsFunctionId = 0x882D5BA1u; -inline constexpr uint32_t kResetDecoderFunctionId = 0x977A59CFu; - -// Hard cap for one enqueue_syndromes request. Enforced at both transport and -// session boundaries so alternate transports cannot bypass allocation and -// packed-length validation. -inline constexpr uint64_t kMaxSyndromeBits = 1u << 20; // 1 M bits - -// Wire magic bytes (from cudaq-realtime spec). -inline constexpr uint32_t kRPCRequestMagic = 0x43555152u; // 'CUQR' -inline constexpr uint32_t kRPCResponseMagic = 0x43555153u; // 'CUQS' - -// Status codes carried in RPCResponse::status. -enum class RpcStatus : int32_t { - OK = 0, - INVALID_DECODER = 1, - BAD_REQUEST = 2, - INTERNAL_ERROR = 3, - NOT_READY = 4, - BUSY = 5, - SYNDROMES_DROPPED = 6, -}; - -// Request header — 24 bytes, packed, little-endian, no padding. -// Layout matches cudaq-realtime RPCHeader exactly. -struct __attribute__((packed)) RPCHeader { - uint32_t magic; ///< kRPCRequestMagic - uint32_t function_id; ///< FNV1a-32 of the callee name - uint32_t arg_len; ///< bytes of payload following this header - uint32_t request_id; ///< caller-assigned; echoed in response - uint64_t ptp_timestamp; ///< PTP send timestamp in ns (0 if unused) -}; -static_assert(sizeof(RPCHeader) == 24, "RPCHeader must be 24 bytes"); - -// Payload structs for each of the three RPCs. -// Layouts mirror decoder_rpc_ids.h without requiring CUDAQ_REALTIME headers. - -struct __attribute__((packed)) EnqueuePayload { - int64_t decoder_id; ///< arg0 - int64_t counter; ///< arg1 - int64_t syndrome_mapping_id; ///< arg2 - int64_t num_syndromes; ///< arg3 (# syndrome bits) - // Trailing: ceil(num_syndromes/8) bit-packed bytes (LSB-first), no pad -}; -static_assert(sizeof(EnqueuePayload) == 32, "EnqueuePayload must be 32 bytes"); - -// Layout per decoder_server_runtime.md: two 8-byte scalars in schema order plus -// a trailing 1-byte bool, NO trailing padding (arg_len = 17 exactly). -struct __attribute__((packed)) GetCorrectionsPayload { - int64_t decoder_id; ///< arg0 - int64_t return_size; ///< arg1 (# correction bits to fetch; the cc.device_call - ///< lowering serializes the OUT std::vector - ///< length here) - uint8_t reset; ///< arg2 (1 = reset decoder after read; trailing bool, - ///< no padding) -}; -static_assert(sizeof(GetCorrectionsPayload) == 17, - "GetCorrectionsPayload must be 17 bytes"); - -struct __attribute__((packed)) ResetPayload { - int64_t decoder_id; ///< arg0 -}; -static_assert(sizeof(ResetPayload) == 8, "ResetPayload must be 8 bytes"); - -// Response header — 24 bytes, packed, little-endian, no padding. -// Layout matches cudaq-realtime RPCResponse exactly. -struct __attribute__((packed)) RPCResponse { - uint32_t magic; ///< kRPCResponseMagic - int32_t status; ///< RpcStatus cast to int32_t; 0 = success - uint32_t result_len; ///< bytes of result payload following (0 on error) - uint32_t request_id; ///< echoed from RPCHeader::request_id - uint64_t ptp_timestamp; ///< echoed from RPCHeader::ptp_timestamp -}; -static_assert(sizeof(RPCResponse) == 24, "RPCResponse must be 24 bytes"); - -// Utility: bytes required to bit-pack N bits. -constexpr size_t bit_packed_bytes(size_t num_bits) noexcept { - return (num_bits + 7) / 8; -} - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/SessionRegistry.cpp b/libs/qec/lib/realtime/decoding-server-cqr/SessionRegistry.cpp deleted file mode 100644 index 718929fb1..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/SessionRegistry.cpp +++ /dev/null @@ -1,107 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#include "SessionRegistry.h" -#include "../realtime_decoding.h" -#include "cudaq/qec/logger.h" -#include "cudaq/qec/realtime/decoding_config.h" - -#include -#include -#include - -namespace cudaq::qec::decoding_server { - -using cudaq::qec::decoding::config::multi_decoder_config; - -/// Build the default single-VP pass-through syndrome mapping table. -/// mapping_id=0 → VP 0 → empty index list (pass-through) -/// -/// An empty index list signals RoundAccumulator to copy bits directly without -/// scatter. This is correct for the nominal per-round enqueue pattern where -/// the caller sends exactly the syndromes for one round and does not need -/// index remapping. An identity-sized index list would force every enqueue -/// to provide exactly syndrome_size bits, which breaks per-round batching. -static SyndromeMappingTable make_default_mapping_table() { - SyndromeMappingTable table; - table[0] = {{}}; // syndrome_mapping_id=0, VP 0, pass-through - return table; -} - -// --------------------------------------------------------------------------- -// SessionRegistry -// --------------------------------------------------------------------------- - -void SessionRegistry::load_from_config(const std::string &yaml_path) { - std::ifstream f(yaml_path); - if (!f.is_open()) - throw std::runtime_error("Cannot open config file: " + yaml_path); - - std::string yaml_str((std::istreambuf_iterator(f)), - std::istreambuf_iterator()); - load_from_config(multi_decoder_config::from_yaml_str(yaml_str), yaml_path); -} - -void SessionRegistry::load_from_config(const multi_decoder_config &config, - const std::string &source_name) { - for (const auto &dc : config.decoders) { - if (dc.id < 0) - throw std::runtime_error("Negative decoder id " + std::to_string(dc.id) + - " in " + source_name); - const uint64_t id = static_cast(dc.id); - if (sessions_.count(id)) - throw std::runtime_error("Duplicate decoder id " + std::to_string(dc.id) + - " in " + source_name); - - // All decoders in one server instance must share the same transport type - // because there is one receive loop per unique transceiver. - if (sessions_.empty()) { - transport_ = dc.transport; - } else if (dc.transport != transport_) { - throw std::runtime_error( - "Mixed transport types in " + source_name + - ": all decoder entries must declare the same transport"); - } - - CUDA_QEC_INFO("SessionRegistry: creating decoder id={} type={}", dc.id, - dc.type); - - auto decoder = cudaq::qec::decoding::host::create_realtime_decoder(dc); - auto session = DecodingSession::create(std::move(decoder), - make_default_mapping_table()); - - // [For follow-up] dc.transport (cpu_roce / gpu_roce) is parsed from YAML - // but not yet used to select a transceiver here. Transport binding requires - // CpuRoceTransceiverAdapter / GpuRoceTransceiverAdapter (gated on - // CUDAQ_REALTIME headers); the split-transport DecodingServer constructor - // is already in place to accept the resulting dispatch map. - session->start_worker(); - sessions_.emplace(id, std::move(session)); - } - - CUDA_QEC_INFO("SessionRegistry: loaded {} decoder session(s)", - sessions_.size()); -} - -DecodingSession &SessionRegistry::get(uint64_t decoder_id) { - auto it = sessions_.find(decoder_id); - if (it == sessions_.end()) - throw std::out_of_range("Unknown decoder_id: " + - std::to_string(decoder_id)); - return *it->second; -} - -const DecodingSession &SessionRegistry::get(uint64_t decoder_id) const { - auto it = sessions_.find(decoder_id); - if (it == sessions_.end()) - throw std::out_of_range("Unknown decoder_id: " + - std::to_string(decoder_id)); - return *it->second; -} - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/SessionRegistry.h b/libs/qec/lib/realtime/decoding-server-cqr/SessionRegistry.h deleted file mode 100644 index f448b3cb8..000000000 --- a/libs/qec/lib/realtime/decoding-server-cqr/SessionRegistry.h +++ /dev/null @@ -1,66 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -#include "DecodingSession.h" -#include "cudaq/qec/realtime/decoding_config.h" - -#include -#include -#include - -namespace cudaq::qec::decoding_server { - -using cudaq::qec::decoding::config::DecoderTransport; - -/// Owns all DecodingSession instances, keyed by uint64_t decoder_id. -/// -/// Populated eagerly at startup from the YAML config. The map is read-only -/// after load_from_config() returns, so no locking is required at runtime. -class SessionRegistry { -public: - /// Parse \p yaml_path and construct one DecodingSession per decoder entry. - /// All decoder entries must declare the same transport type. - /// @throws std::runtime_error on duplicate id, mixed transport types, - /// missing required fields, or decoder init failure. - void load_from_config(const std::string &yaml_path); - - /// Same, from an already-parsed config (the in-process application path, - /// where the config was handed to configure_decoders rather than a file). - /// \p source_name is used in error messages only. - void load_from_config( - const cudaq::qec::decoding::config::multi_decoder_config &config, - const std::string &source_name); - - DecodingSession &get(uint64_t decoder_id); - const DecodingSession &get(uint64_t decoder_id) const; - - /// Transport type shared by all sessions; valid after load_from_config(). - DecoderTransport required_transport() const { return transport_; } - - const std::unordered_map> & - sessions() const { - return sessions_; - } - - /// Stop and join every session's worker thread (each drains its queued - /// items first). Must run while the transports the queued items reply - /// through are still alive; the sessions themselves stay registered so - /// decoder/graph resources can be torn down later in the required order. - void stop_workers() { - for (auto &[id, session] : sessions_) - session->stop_worker(); - } - -private: - std::unordered_map> sessions_; - DecoderTransport transport_{DecoderTransport::cpu_roce}; -}; - -} // namespace cudaq::qec::decoding_server diff --git a/libs/qec/lib/realtime/decoding-server-cqr/decoding_server_cqr.cpp b/libs/qec/lib/realtime/decoding-server-cqr/decoding_server_cqr.cpp index 7a54bd2d3..d2f4ed737 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/decoding_server_cqr.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/decoding_server_cqr.cpp @@ -1,4 +1,4 @@ -/****************************************************************-*- C++ -*-**** +/******************************************************************************* * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * * All rights reserved. * * * @@ -6,295 +6,369 @@ * the terms of the Apache License 2.0 which accompanies this distribution. * ******************************************************************************/ -/// cudaq-realtime (cqr) DeviceCallService plugin for the decoding server. -/// -/// The plugin registers the three default-route RPCs (enqueue_syndromes / -/// get_corrections / reset_decoder) as CUDAQ_DISPATCH_HOST_CALL entries whose -/// handlers are thin delegates into CqrTransceiver::inject(); the actual -/// decoding runs in DecodingServer (one DecodingSession worker thread per -/// configured decoder, so multiple decoders decode concurrently). -/// -/// The decoder configuration comes from, in priority order: -/// 1. the CUDAQ_QEC_DECODER_CONFIG env var (path to a multi_decoder_config -/// YAML) -- the standalone-server path; -/// 2. the last multi_decoder_config passed to -/// cudaq::qec::decoding::config::configure_decoders() in this process -- -/// the in-process (host_dispatch) application path. - -#include "CqrTransceiver.h" -#include "DecodingServer.h" -#include "RpcWireFormat.h" -#include "cudaq/qec/logger.h" -#include "cudaq/qec/realtime/decoding_config.h" +#include "../realtime_decoding.h" #include "cudaq/realtime/daemon/dispatcher/dispatch_kernel_launch.h" #include "cudaq/realtime/device_call_service.h" -#include "../realtime_decoding.h" - +#include #include #include -#include -#include -#include - -extern "C" void cudaqx_qec_decoding_server_shutdown(); #include +#include +#include #include -#include -#include -#include -#include namespace { -using cudaq::qec::decoding_server::CqrTransceiver; -using cudaq::qec::decoding_server::DecodingServer; -using cudaq::qec::decoding_server::kEnqueueSyndromesFunctionId; -using cudaq::qec::decoding_server::kGetCorrectionsFunctionId; -using cudaq::qec::decoding_server::kResetDecoderFunctionId; using cudaq::realtime::DeviceCallDispatchMode; using cudaq::realtime::DeviceCallDispatchTable; using cudaq::realtime::DeviceCallService; using cudaq::realtime::DeviceCallServicePluginInfo; using cudaq::realtime::DeviceCallServiceSession; -static CqrTransceiver *g_transceiver = nullptr; -static std::unique_ptr g_server; -static std::thread g_server_thread; -static std::once_flag g_init_flag; - -// Counts requests dispatched through this service (test hook). -static std::atomic g_service_dispatch_count{0}; - -static void init_server() { - auto t = std::make_unique(); - CqrTransceiver *raw = t.get(); - - if (const char *cfg = std::getenv("CUDAQ_QEC_DECODER_CONFIG"); - cfg && cfg[0] != '\0') { - g_server = std::make_unique(std::move(t), std::string(cfg)); - } else if (const auto config = cudaq::qec::decoding::config:: - last_configured_multi_decoder_config()) { - g_server = std::make_unique(std::move(t), *config); - } else { - throw std::runtime_error( - "decoding-server config not found: set CUDAQ_QEC_DECODER_CONFIG to a " - "multi_decoder_config YAML path, or call " - "cudaq::qec::decoding::config::configure_decoders() before realtime " - "initialization"); - } - // Publish the transceiver only after the server is fully constructed: a - // throwing DecodingServer constructor has already freed the transceiver, and - // dispatch_rpc treats a null g_transceiver as "not serving". - g_transceiver = raw; - g_server_thread = std::thread([] { g_server->run(); }); - // In-process applications never call the explicit shutdown hook the server - // uses; stop the server at exit() so the static g_server_thread is joined - // before static destruction (a still-joinable thread would - // std::terminate, aborting the process and losing buffered stdout). - std::atexit([] { cudaqx_qec_decoding_server_shutdown(); }); +// Realtime function ids are fnv1a_32 of the kernel-facing callee name, matching +// the generic device_call targets emitted by the +// cudaq-qec-realtime-decoding-simulation-cqr device wrappers (and the names +// defined in decoder_server_runtime.md). All three are extern "C", so no name +// mangling is involved. +constexpr std::uint32_t kEnqueueSyndromesFnId = + cudaq::realtime::fnv1a_hash("enqueue_syndromes"); +constexpr std::uint32_t kGetCorrectionsFnId = + cudaq::realtime::fnv1a_hash("get_corrections"); +constexpr std::uint32_t kResetDecoderFnId = + cudaq::realtime::fnv1a_hash("reset_decoder"); + +constexpr std::int32_t kStatusSuccess = 0; +constexpr std::int32_t kStatusInvalidRequest = -1; +constexpr std::int32_t kStatusHandlerException = -2; +constexpr std::int32_t kStatusPayloadTooLarge = -3; +constexpr std::int32_t kStatusResultBufferTooSmall = -5; + +constexpr std::int32_t kHostDispatchDeviceId = 0; +constexpr std::uint32_t kScalarElementCount = 1; + +constexpr std::uint8_t kNoResults = 0; +constexpr std::uint8_t kSingleResult = 1; + +constexpr std::uint8_t kEnqueueDecoderIdArg = 0; +constexpr std::uint8_t kEnqueueSyndromesArg = 1; +constexpr std::uint8_t kEnqueueTagArg = 2; +constexpr std::uint8_t kEnqueueArgCount = 3; + +constexpr std::uint8_t kGetCorrectionsDecoderIdArg = 0; +constexpr std::uint8_t kGetCorrectionsLengthArg = 1; +constexpr std::uint8_t kGetCorrectionsResetArg = 2; +constexpr std::uint8_t kGetCorrectionsArgCount = 3; + +constexpr std::uint8_t kResetDecoderIdArg = 0; +constexpr std::uint8_t kResetDecoderArgCount = 1; + +constexpr std::uint8_t kCorrectionsResult = 0; + +constexpr std::uint8_t kScalarU8Size = sizeof(std::uint8_t); +constexpr std::uint8_t kScalarU64Size = sizeof(std::uint64_t); +constexpr std::size_t kMaxRealtimeVectorLength = 64; + +struct ByteSpan { + const std::uint8_t *data = nullptr; + std::uint64_t size = 0; +}; + +struct EnqueueSyndromesRequest { + std::uint64_t decoder_id = 0; + ByteSpan syndromes; + std::uint64_t tag = 0; +}; + +struct GetCorrectionsRequest { + std::uint64_t decoder_id = 0; + std::uint64_t correction_length = 0; + bool reset = false; +}; + +enum DeviceCallEntryIndex : std::size_t { + kEnqueueSyndromesEntry, + kGetCorrectionsEntry, + kResetDecoderEntry, + kDeviceCallEntryCount +}; + +bool align_offset(std::size_t &offset, std::size_t alignment, + std::size_t arg_len) { + if (alignment <= 1) + return offset <= arg_len; + const auto addend = alignment - 1; + if (offset > std::numeric_limits::max() - addend) + return false; + offset = (offset + addend) & ~addend; + return offset <= arg_len; } -// --------------------------------------------------------------------------- -// CUDAQ handler functions — thin delegates to CqrTransceiver::inject() -// --------------------------------------------------------------------------- - -// Write an error RPCResponse into tx_slot (handler-level failures must not -// propagate into the transport dispatcher loop). -constexpr int32_t kStatusHandlerException = static_cast( - cudaq::qec::decoding_server::RpcStatus::INTERNAL_ERROR); - -static void write_error_response(const void *rx_slot, void *tx_slot, - std::size_t slot_size, int32_t status) { - if (!tx_slot || !rx_slot || slot_size < sizeof(cudaq::realtime::RPCHeader)) - return; - const auto *req = static_cast(rx_slot); - auto *resp = static_cast(tx_slot); - resp->status = status; - resp->result_len = 0; - resp->request_id = req->request_id; - resp->ptp_timestamp = req->ptp_timestamp; - __atomic_store_n(reinterpret_cast(tx_slot), - cudaq::realtime::RPC_MAGIC_RESPONSE, __ATOMIC_RELEASE); +template +bool read_scalar(const std::uint8_t *payload, std::size_t arg_len, + std::size_t &offset, T &value) { + if (!align_offset(offset, alignof(T), arg_len) || + sizeof(T) > arg_len - offset) + return false; + std::memcpy(&value, payload + offset, sizeof(T)); + offset += sizeof(T); + return true; } -// --save_syndrome support: the served path bypasses host::enqueue_syndromes -// (where capture used to hook), so replicate its capture here -- unpack the -// wire's LSB-first bits and repack MSB-first, byte-identical to the host -// path's saved-syndrome format. -static void capture_enqueue_syndromes(const void *rx_slot, - std::size_t slot_size) { - auto callback = cudaq::qec::decoding::host::_get_syndrome_capture_callback(); - if (!callback) - return; - cudaq::qec::decoding_server::detail::CqrEnqueueFrameView request; - if (!cudaq::qec::decoding_server::detail::parse_cqr_enqueue_frame( - rx_slot, slot_size, request)) - return; - std::vector packed(request.byte_count, 0); - for (uint64_t i = 0; i < request.num_syndromes; ++i) - if ((request.packed_bits[i / 8] >> (i % 8)) & 1u) - packed[i / 8] |= static_cast(1u << (7 - (i % 8))); - callback(packed.data(), packed.size()); +bool read_stdvec_i1(const std::uint8_t *payload, std::size_t arg_len, + std::size_t &offset, ByteSpan &span) { + // CUDA-Q realtime lowering serializes stdvec as a uint64 length followed + // by one byte per bool element. + std::uint64_t length = 0; + if (!read_scalar(payload, arg_len, offset, length)) + return false; + if (length > static_cast(arg_len - offset)) + return false; + span = {payload + offset, length}; + offset += static_cast(length); + return true; } -// The server is constructed lazily on the first RPC (the in-process -// application path configures decoders AFTER the realtime channel — and -// with it this dispatch session — is created); the server path instead -// initializes eagerly at session creation via CUDAQ_QEC_DECODER_CONFIG so -// slow decoder construction happens before its READY line. -static void dispatch_rpc(const void *rx_slot, void *tx_slot, - std::size_t slot_size, uint32_t function_id) { +// Counts validated requests this host-dispatch service has handled. Lets a test +// confirm the device_call actually traversed the host-dispatch ring to the +// service (HOP1) rather than resolving to a direct host trampoline. Exposed via +// cudaqx_qec_device_call_dispatch_count() below. +std::atomic g_service_dispatch_count{0}; + +bool read_request_payload(const void *rx_slot, std::size_t slot_size, + const cudaq::realtime::RPCHeader *&request, + const std::uint8_t *&payload, std::size_t &arg_len) { + if (!rx_slot || slot_size < sizeof(cudaq::realtime::RPCHeader)) + return false; + + request = static_cast(rx_slot); + if (request->magic != cudaq::realtime::RPC_MAGIC_REQUEST) + return false; + + arg_len = request->arg_len; + if (arg_len > slot_size - sizeof(cudaq::realtime::RPCHeader)) + return false; + + payload = static_cast(rx_slot) + + sizeof(cudaq::realtime::RPCHeader); g_service_dispatch_count.fetch_add(1, std::memory_order_relaxed); + return true; +} + +bool read_enqueue_syndromes_request(const void *rx_slot, std::size_t slot_size, + EnqueueSyndromesRequest &out) { + const cudaq::realtime::RPCHeader *request = nullptr; + const std::uint8_t *payload = nullptr; + std::size_t arg_len = 0; + if (!read_request_payload(rx_slot, slot_size, request, payload, arg_len)) + return false; + + std::size_t offset = 0; + return read_scalar(payload, arg_len, offset, out.decoder_id) && + read_stdvec_i1(payload, arg_len, offset, out.syndromes) && + read_scalar(payload, arg_len, offset, out.tag) && offset == arg_len; +} + +bool read_get_corrections_request(const void *rx_slot, std::size_t slot_size, + GetCorrectionsRequest &out) { + const cudaq::realtime::RPCHeader *request = nullptr; + const std::uint8_t *payload = nullptr; + std::size_t arg_len = 0; + if (!read_request_payload(rx_slot, slot_size, request, payload, arg_len)) + return false; + + std::size_t offset = 0; + std::uint8_t reset = 0; + if (!read_scalar(payload, arg_len, offset, out.decoder_id) || + !read_scalar(payload, arg_len, offset, out.correction_length) || + !read_scalar(payload, arg_len, offset, reset) || offset != arg_len) + return false; + + out.reset = reset != 0; + return true; +} + +bool read_reset_decoder_request(const void *rx_slot, std::size_t slot_size, + std::uint64_t &decoder_id) { + const cudaq::realtime::RPCHeader *request = nullptr; + const std::uint8_t *payload = nullptr; + std::size_t arg_len = 0; + if (!read_request_payload(rx_slot, slot_size, request, payload, arg_len)) + return false; + + std::size_t offset = 0; + return read_scalar(payload, arg_len, offset, decoder_id) && offset == arg_len; +} + +void write_response(void *tx_slot, const void *rx_slot, std::int32_t status, + std::uint32_t result_len = 0) { + const auto *request = + static_cast(rx_slot); + auto *response = static_cast(tx_slot); + response->status = status; + response->result_len = result_len; + response->request_id = request ? request->request_id : 0; + response->ptp_timestamp = request ? request->ptp_timestamp : 0; + __atomic_store_n(&response->magic, cudaq::realtime::RPC_MAGIC_RESPONSE, + __ATOMIC_RELEASE); +} + +void enqueue_syndromes_host(const void *rx_slot, void *tx_slot, + std::size_t slot_size) { try { - std::call_once(g_init_flag, init_server); - // g_transceiver is null if init_server failed or after shutdown(). - // g_init_flag is not resettable, so call_once won't retry after shutdown. - if (!g_transceiver) { - write_error_response(rx_slot, tx_slot, slot_size, - kStatusHandlerException); + EnqueueSyndromesRequest request; + if (!tx_slot || + !read_enqueue_syndromes_request(rx_slot, slot_size, request)) { + if (tx_slot && rx_slot) + write_response(tx_slot, rx_slot, kStatusInvalidRequest); + return; + } + + if (request.syndromes.size > kMaxRealtimeVectorLength) { + write_response(tx_slot, rx_slot, kStatusPayloadTooLarge); return; } - if (function_id == kEnqueueSyndromesFunctionId) - capture_enqueue_syndromes(rx_slot, slot_size); - g_transceiver->inject(rx_slot, tx_slot, slot_size, function_id); - } catch (const std::exception &e) { - // Log via the non-throwing cudaq::qec::error() free function, NOT the - // CUDA_QEC_ERROR macro: the macro throws, and an exception escaping this - // handler into the transport dispatcher loop would terminate the process - // instead of returning the error response written below. - cudaq::qec::error("decoding-server RPC failed: {}", e.what()); - write_error_response(rx_slot, tx_slot, slot_size, kStatusHandlerException); + + std::array syndrome{}; + if (request.syndromes.size != 0) + std::memcpy(syndrome.data(), request.syndromes.data, + request.syndromes.size); + cudaq::qec::decoding::host::enqueue_syndromes( + static_cast(request.decoder_id), syndrome.data(), + static_cast(request.syndromes.size), request.tag); + write_response(tx_slot, rx_slot, kStatusSuccess); } catch (...) { - write_error_response(rx_slot, tx_slot, slot_size, kStatusHandlerException); + if (tx_slot && rx_slot) + write_response(tx_slot, rx_slot, kStatusHandlerException); } } -void enqueue_syndromes_host(const void *rx_slot, void *tx_slot, - std::size_t slot_size) { - dispatch_rpc(rx_slot, tx_slot, slot_size, kEnqueueSyndromesFunctionId); -} - void get_corrections_host(const void *rx_slot, void *tx_slot, std::size_t slot_size) { - dispatch_rpc(rx_slot, tx_slot, slot_size, kGetCorrectionsFunctionId); -} + try { + GetCorrectionsRequest request; + if (!tx_slot || + !read_get_corrections_request(rx_slot, slot_size, request)) { + if (tx_slot && rx_slot) + write_response(tx_slot, rx_slot, kStatusInvalidRequest); + return; + } -void reset_decoder_host(const void *rx_slot, void *tx_slot, - std::size_t slot_size) { - dispatch_rpc(rx_slot, tx_slot, slot_size, kResetDecoderFunctionId); -} + if (request.correction_length > kMaxRealtimeVectorLength) { + write_response(tx_slot, rx_slot, kStatusPayloadTooLarge); + return; + } -// --------------------------------------------------------------------------- -// DeviceCallService plugin -// --------------------------------------------------------------------------- - -// The schema entries below register under the SAME function IDs the handlers -// and CqrTransceiver route on (the kXFunctionId constants from -// RpcWireFormat.h); these asserts pin them to the fnv1a hashes of the RPC -// names so a rename cannot silently desynchronize registration from routing. -static_assert(kEnqueueSyndromesFunctionId == - cudaq::realtime::fnv1a_hash("enqueue_syndromes")); -static_assert(kGetCorrectionsFunctionId == - cudaq::realtime::fnv1a_hash("get_corrections")); -static_assert(kResetDecoderFunctionId == - cudaq::realtime::fnv1a_hash("reset_decoder")); - -constexpr int32_t kHostDispatchDeviceId = 0; -constexpr uint8_t kNoResults = 0; -constexpr uint8_t kSingleResult = 1; -constexpr uint8_t kScalarU8Size = sizeof(uint8_t); -constexpr uint8_t kScalarU64Size = sizeof(uint64_t); - -// Wire argument order per decoder_server_runtime.md: fixed-size scalars -// first, the variable-length bit-packed byte array last. -constexpr std::uint8_t kEnqueueDecoderIdArg = 0; -constexpr std::uint8_t kEnqueueCounterArg = 1; -constexpr std::uint8_t kEnqueueMappingIdArg = 2; -constexpr std::uint8_t kEnqueueSyndromeBitsArg = 3; -constexpr std::uint8_t kEnqueueArgCount = 4; + const auto result_len = + static_cast(request.correction_length); + if (sizeof(cudaq::realtime::RPCResponse) + result_len > slot_size) { + write_response(tx_slot, rx_slot, kStatusResultBufferTooSmall); + return; + } -constexpr std::uint8_t kGetCorrectionsDecoderIdArg = 0; -constexpr std::uint8_t kGetCorrectionsReturnSizeArg = 1; -constexpr std::uint8_t kGetCorrectionsResetArg = 2; -constexpr std::uint8_t kGetCorrectionsArgCount = 3; + std::array corrections{}; + cudaq::qec::decoding::host::get_corrections( + static_cast(request.decoder_id), corrections.data(), + static_cast(request.correction_length), request.reset); -constexpr std::uint8_t kResetDecoderIdArg = 0; -constexpr std::uint8_t kResetDecoderArgCount = 1; + auto *result = static_cast(tx_slot) + + sizeof(cudaq::realtime::RPCResponse); + if (result_len != 0) + std::memcpy(result, corrections.data(), result_len); + write_response(tx_slot, rx_slot, kStatusSuccess, result_len); + } catch (...) { + if (tx_slot && rx_slot) + write_response(tx_slot, rx_slot, kStatusHandlerException); + } +} -constexpr std::uint8_t kCorrectionsResult = 0; +void reset_decoder_host(const void *rx_slot, void *tx_slot, + std::size_t slot_size) { + try { + std::uint64_t decoder_id = 0; + if (!tx_slot || + !read_reset_decoder_request(rx_slot, slot_size, decoder_id)) { + if (tx_slot && rx_slot) + write_response(tx_slot, rx_slot, kStatusInvalidRequest); + return; + } -enum DeviceCallEntryIndex : std::size_t { - kEnqueueSyndromesEntry, - kGetCorrectionsEntry, - kResetDecoderEntry, - kDeviceCallEntryCount -}; + cudaq::qec::decoding::host::reset_decoder( + static_cast(decoder_id)); + write_response(tx_slot, rx_slot, kStatusSuccess); + } catch (...) { + if (tx_slot && rx_slot) + write_response(tx_slot, rx_slot, kStatusHandlerException); + } +} + +void set_scalar(cudaq_type_desc_t &desc, std::uint8_t type_id, + std::uint32_t size_bytes) { + desc = {}; + desc.type_id = type_id; + desc.size_bytes = size_bytes; + desc.num_elements = kScalarElementCount; +} -static void set_u64(cudaq_type_desc_t &d) { - d = {}; - d.type_id = CUDAQ_TYPE_INT64; - d.size_bytes = kScalarU64Size; - d.num_elements = 1; +void set_u64(cudaq_type_desc_t &desc) { + set_scalar(desc, CUDAQ_TYPE_INT64, kScalarU64Size); } -static void set_u8(cudaq_type_desc_t &d) { - d = {}; - d.type_id = CUDAQ_TYPE_UINT8; - d.size_bytes = kScalarU8Size; - d.num_elements = 1; +void set_u8(cudaq_type_desc_t &desc) { + set_scalar(desc, CUDAQ_TYPE_UINT8, kScalarU8Size); } -// Syndrome/correction bits cross the wire bit-packed (LSB-first), so the -// argument type is CUDAQ_TYPE_BIT_PACKED -- matching the realtime device_call -// lowering for std::vector (cudaq PR 4816) -- rather than the old -// CUDAQ_TYPE_ARRAY_UINT8 stand-in used before that lowering existed. -static void set_bit_packed(cudaq_type_desc_t &d) { - d = {}; - d.type_id = CUDAQ_TYPE_BIT_PACKED; +void set_array_u8(cudaq_type_desc_t &desc) { + desc = {}; + desc.type_id = CUDAQ_TYPE_ARRAY_UINT8; } -static void configure_entry(cudaq_function_entry_t &e, uint32_t fn_id, - cudaq_host_rpc_fn_t handler, uint8_t num_args, - uint8_t num_results) { - e = {}; - e.handler.host_fn = handler; - e.function_id = fn_id; - e.dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; - e.schema.num_args = num_args; - e.schema.num_results = num_results; +void configure_entry(cudaq_function_entry_t &entry, std::uint32_t function_id, + cudaq_host_rpc_fn_t handler, std::uint8_t num_args, + std::uint8_t num_results) { + entry = {}; + entry.handler.host_fn = handler; + entry.function_id = function_id; + entry.dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; + entry.schema.num_args = num_args; + entry.schema.num_results = num_results; } -static std::array -make_entries() { +// Registers the fixed default-route RPC handlers (enqueue_syndromes / +// get_corrections / reset_decoder), all as CUDAQ_DISPATCH_HOST_CALL entries +// that decode on the CPU via the host API. +// +// TODO(decoding-server): a more complete decoding server should build the +// function table per configured decoder rather than as this fixed CPU set. Each +// decoder instance would get its own configure_entry establishing the correct +// processing mechanism -- CUDAQ_DISPATCH_HOST_CALL for CPU/host decoders, or +// CUDAQ_DISPATCH_GRAPH_LAUNCH for GPU decoders driven by a captured CUDA graph +// (see decoder_server_runtime.md "per-decoder alternative dispatch units"). +// That is what would let the device_call host-dispatch path (this service) +// dispatch straight to the right per-decoder mechanism, superseding the +// separate qec_realtime_session decode ring entirely. +std::array make_entries() { std::array entries{}; - // enqueue_syndromes: 4-arg spec format per decoder_server_runtime.md. - // decoder_id, counter, syndrome_mapping_id (scalars) + syndrome_bits - // (bit_packed: element-count prefix == num_syndromes, then LSB-first bits). - auto &eq = entries[kEnqueueSyndromesEntry]; - configure_entry(eq, kEnqueueSyndromesFunctionId, enqueue_syndromes_host, + auto &enqueue_entry = entries[kEnqueueSyndromesEntry]; + configure_entry(enqueue_entry, kEnqueueSyndromesFnId, enqueue_syndromes_host, kEnqueueArgCount, kNoResults); - set_u64(eq.schema.args[kEnqueueDecoderIdArg]); - set_u64(eq.schema.args[kEnqueueCounterArg]); - set_u64(eq.schema.args[kEnqueueMappingIdArg]); - set_bit_packed(eq.schema.args[kEnqueueSyndromeBitsArg]); - - // get_corrections: 3-arg spec format per decoder_server_runtime.md. - // decoder_id (scalar) + corrections (OUT std::vector: the request - // carries its length as return_size) + reset (scalar). - auto &gc = entries[kGetCorrectionsEntry]; - configure_entry(gc, kGetCorrectionsFunctionId, get_corrections_host, + set_u64(enqueue_entry.schema.args[kEnqueueDecoderIdArg]); + set_array_u8(enqueue_entry.schema.args[kEnqueueSyndromesArg]); + set_u64(enqueue_entry.schema.args[kEnqueueTagArg]); + + auto &get_entry = entries[kGetCorrectionsEntry]; + configure_entry(get_entry, kGetCorrectionsFnId, get_corrections_host, kGetCorrectionsArgCount, kSingleResult); - set_u64(gc.schema.args[kGetCorrectionsDecoderIdArg]); - set_u64(gc.schema.args[kGetCorrectionsReturnSizeArg]); - set_u8(gc.schema.args[kGetCorrectionsResetArg]); - set_bit_packed(gc.schema.results[kCorrectionsResult]); + set_u64(get_entry.schema.args[kGetCorrectionsDecoderIdArg]); + set_u64(get_entry.schema.args[kGetCorrectionsLengthArg]); + set_u8(get_entry.schema.args[kGetCorrectionsResetArg]); + set_array_u8(get_entry.schema.results[kCorrectionsResult]); - auto &rd = entries[kResetDecoderEntry]; - configure_entry(rd, kResetDecoderFunctionId, reset_decoder_host, + auto &reset_entry = entries[kResetDecoderEntry]; + configure_entry(reset_entry, kResetDecoderFnId, reset_decoder_host, kResetDecoderArgCount, kNoResults); - set_u64(rd.schema.args[kResetDecoderIdArg]); + set_u64(reset_entry.schema.args[kResetDecoderIdArg]); return entries; } @@ -302,97 +376,51 @@ make_entries() { class QecDeviceCallSession : public DeviceCallServiceSession { public: QecDeviceCallSession() { - table_.mode = DeviceCallDispatchMode::Host; - table_.entries = entries_.data(); - table_.count = entries_.size(); - table_.deviceId = kHostDispatchDeviceId; - table_.mailbox = nullptr; + table.mode = DeviceCallDispatchMode::Host; + table.entries = entries.data(); + table.count = entries.size(); + table.deviceId = kHostDispatchDeviceId; + table.mailbox = nullptr; } const DeviceCallDispatchTable &dispatchTable() const noexcept override { - return table_; + return table; } private: - std::array entries_ = + std::array entries = make_entries(); - DeviceCallDispatchTable table_; + DeviceCallDispatchTable table; }; class QecDeviceCallService : public DeviceCallService { public: std::unique_ptr createDispatchSession(DeviceCallDispatchMode mode) override { + // This service registers CUDAQ_DISPATCH_HOST_CALL handlers only. if (mode != DeviceCallDispatchMode::Host) return nullptr; - // Server path: the config path is in the environment, so build the - // decoder sessions NOW (before the server's READY line). The in-process - // application path has not called configure_decoders yet at this point; - // it initializes lazily on the first RPC (see dispatch_rpc). - if (const char *cfg = std::getenv("CUDAQ_QEC_DECODER_CONFIG"); - cfg && cfg[0] != '\0') { - try { - std::call_once(g_init_flag, init_server); - } catch (const std::exception &e) { - // CUDAQ core does not expect plugin session creation to throw; a - // propagating exception would escape the channel-setup path and - // terminate. Report the config/decoder failure and decline the - // session instead. - cudaq::qec::error( - "decoding-server init failed (CUDAQ_QEC_DECODER_CONFIG={}): {}", - cfg, e.what()); - return nullptr; - } - } return std::make_unique(); } }; QecDeviceCallService g_service; + DeviceCallService *get_service() { return &g_service; } } // namespace -extern "C" __attribute__((visibility("default"))) -cudaq::realtime::DeviceCallServicePluginInfo -cudaqGetDeviceCallServicePluginInfo() { - return {"cudaq-qec-realtime-device-call", &get_service}; -} - extern "C" __attribute__((visibility("default"))) void cudaqx_qec_realtime_device_call_service_force_link() {} -extern "C" __attribute__((visibility("default"))) uint64_t +// Test hook: number of requests this service has dispatched. Non-zero only if +// device_calls were routed through the host-dispatch ring to this service. +extern "C" __attribute__((visibility("default"))) std::uint64_t cudaqx_qec_device_call_dispatch_count() { return g_service_dispatch_count.load(std::memory_order_relaxed); } -/// High-water mark of simultaneously-busy DecodingSession workers -- the -/// server's concurrency evidence for multi-logical-qubit tests. -extern "C" __attribute__((visibility("default"))) uint64_t -cudaqx_qec_decoding_server_max_concurrent() { - return cudaq::qec::decoding_server::max_concurrent_busy_sessions(); -} - -/// Per-decoder session counters (decodes/enqueues/...), one stdout line per -/// decoder. Test/diagnostic evidence; callers gate on the -/// QEC_DECODING_SERVER_STATS environment variable. -extern "C" __attribute__((visibility("default"))) void -cudaqx_qec_decoding_server_print_stats() { - if (g_server) - g_server->print_session_stats(); -} - -/// Stop the DecodingServer receive loop and join its thread. The server calls -/// this before exiting; without it the static g_server_thread would still be -/// joinable at static destruction (std::terminate). -extern "C" __attribute__((visibility("default"))) void -cudaqx_qec_decoding_server_shutdown() { - if (g_server) { - g_server->stop(); - if (g_server_thread.joinable()) - g_server_thread.join(); - g_server.reset(); - g_transceiver = nullptr; - } +extern "C" __attribute__((visibility("default"))) DeviceCallServicePluginInfo +cudaqGetDeviceCallServicePluginInfo() { + return {"cudaq-qec-realtime-device-call", &get_service}; } diff --git a/libs/qec/lib/realtime/gpu_kernels.cu b/libs/qec/lib/realtime/gpu_kernels.cu index 14daf3ddb..838cff453 100644 --- a/libs/qec/lib/realtime/gpu_kernels.cu +++ b/libs/qec/lib/realtime/gpu_kernels.cu @@ -52,7 +52,7 @@ __device__ void postprocess_observable( } } - corrections[observable_idx] ^= result; + corrections[observable_idx] = result; } __device__ void preprocess_all(const uint8_t *__restrict__ measurements, diff --git a/libs/qec/lib/realtime/qec_realtime_session.cpp b/libs/qec/lib/realtime/qec_realtime_session.cpp index e949bba82..584359bb0 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.cpp +++ b/libs/qec/lib/realtime/qec_realtime_session.cpp @@ -9,114 +9,27 @@ #ifdef CUDAQ_REALTIME_ROOT #include "qec_realtime_session.h" - #include "../hardware_guards.h" #include "cudaq/qec/logger.h" #include "cudaq/qec/realtime/decoder_rpc_ids.h" -#include "cudaq/qec/realtime/graph_resources.h" -#include "cudaq/realtime/daemon/dispatcher/dispatch_kernel_launch.h" #include #include -#include #include -#include -#include #include #include namespace cudaq::qec::realtime { - namespace { using Decoders = std::vector>; -//============================================================================== -// DEVICE-mode helpers -//============================================================================== - -// Resolves a host-side C-ABI shim `cudaqx_qec_realtime_dispatch_populate_*` -// at runtime via dlsym(RTLD_DEFAULT, ...). These shims are defined in -// libcudaq-qec-realtime-cudevice.a and only enter the process when the final -// executable absorbs that static archive (typically via the -// `qec_realtime_app_link_options()` CMake helper). Resolving by name rather -// than by direct symbol reference keeps libcudaq-qec-realtime-decoding.so free -// of unresolved C-ABI symbols, so it can be safely dlopen'd from consumers that -// do NOT link the cudevice archive (notably the Python extension). Any such -// consumer that tries to actually USE the device dispatch path lands here, does -// not find the symbol, and surfaces a clean runtime_error with actionable -// linker guidance. -using populate_device_entry_fn = void (*)(void *); -populate_device_entry_fn resolve_populate_shim(const char *symbol_name) { - void *sym = ::dlsym(RTLD_DEFAULT, symbol_name); - return reinterpret_cast(sym); -} - -// The device-graph dispatch API (cudaq_create_dispatch_graph_regular / -// cudaq_launch_dispatch_graph / cudaq_destroy_dispatch_graph) lives in -// libcudaq-realtime-dispatch.a -- relocatable device code the host executable -// absorbs and device-links (see qec_realtime_app_link_options()). It must be -// resolved from the SAME image as the populate shims above: the dispatch -// kernel invokes the decoder's __device__ function pointers captured by those -// shims, and a device function pointer is only valid inside the CUDA module -// (device-link unit) that produced it. If this .so carried its own copy of -// the archive, its dispatch kernel would live in a different CUDA module than -// the executable's decoder handlers and the DEVICE_CALL would trap with -// cudaErrorIllegalInstruction. Resolving via dlsym(RTLD_DEFAULT, ...) binds -// to the executable's copy (exported via --export-dynamic + -// CUDAQ_REALTIME_DISPATCH_API default visibility), keeping kernel and handlers -// in one module -- and keeps this .so free of undefined C-ABI symbols for -// consumers that never touch the device path. -using create_dispatch_graph_fn_t = cudaError_t (*)( - volatile std::uint64_t *, volatile std::uint64_t *, std::uint8_t *, - std::uint8_t *, std::size_t, std::size_t, cudaq_function_entry_t *, - std::size_t, void *, volatile int *, std::uint64_t *, std::size_t, - std::uint32_t, std::uint32_t, cudaGraphExec_t, cudaStream_t, - cudaq_dispatch_graph_context **); -using launch_dispatch_graph_fn_t = - cudaError_t (*)(cudaq_dispatch_graph_context *, cudaStream_t); - -template -FnT resolve_dispatch_graph_api(const char *symbol_name) { - void *sym = ::dlsym(RTLD_DEFAULT, symbol_name); - if (!sym) - throw std::runtime_error( - std::string("qec_realtime_session::initialize: ") + symbol_name + - " not found via dlsym(RTLD_DEFAULT, ...). The host executable must " - "absorb libcudaq-realtime-dispatch.a and link with --export-dynamic " - "(see qec_realtime_app_link_options())."); - return reinterpret_cast(sym); -} - -// Pinned mapped flags + pinned mapped data, with the device pointer obtained -// via UVA so the GPU dispatcher can read the same backing. -bool allocate_pinned_mapped(std::size_t bytes, void **host_out, - void **device_out) { - void *h = nullptr; - if (cudaHostAlloc(&h, bytes, cudaHostAllocMapped) != cudaSuccess) - return false; - void *d = nullptr; - if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) { - cudaFreeHost(h); - return false; - } - std::memset(h, 0, bytes); - *host_out = h; - *device_out = d; - return true; -} - -//============================================================================== -// HOST-mode helpers (two-ring CUDAQ_DISPATCH_HOST_CALL handlers) -//============================================================================== - // The HOST_CALL handlers below are plain C function pointers with no // user-context argument, so the active decoder table must be reachable from a -// process-global. It is published/cleared by a HOST-mode initialize()/ -// finalize() and read concurrently by the host dispatcher thread, hence the -// atomic. Only one HOST-mode session may be live at a time (enforced in -// initialize()). +// process-global. It is published/cleared by initialize()/finalize() and read +// concurrently by the host dispatcher thread, hence the atomic. Only one +// session may be live at a time (enforced in initialize()). std::atomic g_active_decoders{nullptr}; cudaq::qec::decoder *get_decoder_or_throw(std::int64_t decoder_id) { @@ -129,27 +42,13 @@ cudaq::qec::decoder *get_decoder_or_throw(std::int64_t decoder_id) { return (*decoders)[static_cast(decoder_id)].get(); } -// Point the calling thread at the decoder's pinned CUDA device before work -// that allocates or launches on it. Set-and-leave (no restore): one -// dispatcher thread serves all decoders, so the thread simply converges to -// the device of the decoder it is currently serving; cudaSetDevice on an -// already-current device is a cheap no-op. -static void apply_decoder_cuda_device(cudaq::qec::decoder *dec) { - if (!dec) +void apply_decoder_cuda_device(cudaq::qec::decoder *decoder) { + if (!decoder) return; - // Throws on failure (fail fast): host dispatch surfaces it as an error - // response and graph initialization aborts, rather than continuing on - // whichever device happened to be current. cudaq::qec::detail_affinity::set_cuda_device_for_decode( - dec->get_cuda_device_id()); + decoder->get_cuda_device_id()); } -// Two-ring response writer: the request stays in `rx_slot` (read-only); the -// response is written into the distinct `tx_slot`. The preserved header fields -// (request_id, ptp_timestamp) must be echoed explicitly from rx to tx. The -// caller (cudaq_host_dispatcher_loop::handle_host_call) publishes tx_flags -// AFTER the handler returns; the handler only needs to write the response body -// + header and release-store the magic. void write_response(void *tx_slot, const void *rx_slot, std::int32_t status, std::uint32_t result_len = 0) { const auto *request = @@ -186,8 +85,9 @@ void enqueue_syndromes_host(const void *rx_slot, void *tx_slot, return; } const auto num_syndromes = static_cast(body->num_syndromes); - const std::size_t expected_arg_len = sizeof(rpc::EnqueueRequestPayload) + - rpc::bit_packed_bytes(num_syndromes); + const std::size_t expected_arg_len = + rpc::align_to_8(sizeof(rpc::EnqueueRequestPayload) + + rpc::bit_packed_bytes(num_syndromes)); if (header->arg_len != expected_arg_len || sizeof(cudaq::realtime::RPCHeader) + expected_arg_len > slot_size) { write_response(tx_slot, rx_slot, -4); @@ -240,13 +140,14 @@ void get_corrections_host(const void *rx_slot, void *tx_slot, } auto *decoder = get_decoder_or_throw(body->decoder_id); + apply_decoder_cuda_device(decoder); const auto return_size = static_cast(body->return_size); if (return_size > decoder->get_num_observables()) { write_response(tx_slot, rx_slot, -4); return; } - // result_len = ceil(R/8) exactly per decoder_server_runtime.md (no pad). - const std::size_t result_len = rpc::bit_packed_bytes(return_size); + const std::size_t result_len = + rpc::align_to_8(rpc::bit_packed_bytes(return_size)); if (sizeof(cudaq::realtime::RPCResponse) + result_len > slot_size) { write_response(tx_slot, rx_slot, -5); return; @@ -261,7 +162,8 @@ void get_corrections_host(const void *rx_slot, void *tx_slot, } if (body->reset != 0) decoder->clear_corrections(); - write_response(tx_slot, rx_slot, 0, static_cast(result_len)); + write_response(tx_slot, rx_slot, 0, + static_cast(result_len)); } catch (...) { write_response(tx_slot, rx_slot, -2); } @@ -290,713 +192,140 @@ void reset_decoder_host(const void *rx_slot, void *tx_slot, std::size_t) { } // namespace -//============================================================================== -// ctor / dtor -//============================================================================== - -qec_realtime_session::qec_realtime_session( - std::vector> &decoders, - cudaq_dispatch_launch_fn_t device_launch_fn) - : decoders_(decoders), device_launch_fn_(device_launch_fn) {} - -qec_realtime_session::~qec_realtime_session() { - // Best-effort teardown. finalize() is null-safe at every step (each resource - // has its own guard), so calling it from a never-fully-initialized or - // already-finalized session is a no-op beyond the trace message. - finalize(); -} - -//============================================================================== -// classify_mode() -//============================================================================== - -void qec_realtime_session::classify_mode() { - bool any_graph = false; - bool any_host = false; - std::size_t non_null = 0; - for (auto &decoder : decoders_) { - if (!decoder) - continue; - ++non_null; - if (decoder->supports_graph_dispatch()) - any_graph = true; - else - any_host = true; - } - - if (non_null == 0) - throw std::runtime_error( - "qec_realtime_session::initialize: no (non-null) decoders to serve"); - - if (any_graph && any_host) - throw std::runtime_error( - "qec_realtime_session::initialize: mixed decoder set -- some decoders " - "support graph dispatch (DEVICE mode) and some do not (HOST mode). A " - "single session must be homogeneous: the host loop resolves a slot to " - "a function table entry by function_id alone, so a GRAPH_LAUNCH and a " - "HOST_CALL enqueue sharing kEnqueueSyndromesFunctionId would collide. " - "Use one decoder per session (or a homogeneous decoder set)."); +qec_realtime_session::qec_realtime_session(Decoders &decoders) + : decoders_(decoders) {} - device_mode_ = any_graph; - - if (device_mode_ && !device_launch_fn_) - throw std::runtime_error( - "qec_realtime_session::initialize: DEVICE mode requires a non-null " - "device_launch_fn (typically &cudaq_launch_dispatch_kernel_regular " - "from libcudaq-realtime-dispatch), but the session was constructed " - "without one."); -} - -//============================================================================== -// initialize() -//============================================================================== +qec_realtime_session::~qec_realtime_session() { finalize(); } void qec_realtime_session::initialize() { if (initialized_) return; - - classify_mode(); - - // Reset the monotonic producer cursor so it starts in lockstep with the - // strict-FIFO consumer (both begin at slot 0). - producer_cursor_ = 0; - - if (device_mode_) { - // The device-graph scheduler relies on device-side graph launch, which - // requires compute capability 9.0+ (Hopper). Below sm_90 the dispatch - // kernel's TRIGGER_GRAPH interception is compiled out and the enqueue RPC - // would surface the raw sentinel (0x12A6E5) as a non-zero status -- fail - // fast with an actionable message instead. - { - int device = 0; - cudaDeviceProp prop{}; - if (cudaGetDevice(&device) != cudaSuccess || - cudaGetDeviceProperties(&prop, device) != cudaSuccess) - throw std::runtime_error( - "qec_realtime_session::initialize: unable to query the active " - "CUDA device for the DEVICE-mode compute-capability check"); - if (prop.major < 9) - throw std::runtime_error( - "qec_realtime_session::initialize: DEVICE mode (the device-graph " - "scheduler) requires device-side graph launch, i.e. compute " - "capability 9.0+ (Hopper); found " + - std::to_string(prop.major) + "." + std::to_string(prop.minor) + - ". Use a CPU decoder (HOST mode) or a Hopper+ GPU."); - } - - // Be tolerant of being called before any CUDA setup. The pinned - // allocations below require cudaDeviceMapHost on the active device. - cudaError_t flags_err = cudaSetDeviceFlags(cudaDeviceMapHost); - if (flags_err != cudaSuccess && flags_err != cudaErrorSetOnActiveProcess) - throw std::runtime_error( - std::string("qec_realtime_session::initialize: " - "cudaSetDeviceFlags(cudaDeviceMapHost) failed: ") + - cudaGetErrorString(flags_err)); - } else { - // HOST mode: claim the process-global decoder table up front. With a - // single global, a second HOST-mode initialize() would hijack decoder-id - // resolution for the first session's host loop. - Decoders *expected = nullptr; - if (!g_active_decoders.compare_exchange_strong(expected, &decoders_, - std::memory_order_acq_rel)) - throw std::runtime_error( - "qec_realtime_session::initialize: another HOST_CALL session is " - "already active; concurrent HOST-mode sessions are not supported"); - } - - // Everything below acquires resources, so it must be transactional. - // finalize() is null-safe at every step, so we can roll a half-built session - // back from any throw. + // Reject a second concurrent session. With a single process-global decoder + // table, a second initialize() would hijack decoder-id resolution for the + // first session's host loop. Claim the global atomically up front; the only + // alternative (threading session context through the host handler path) would + // require a cuda-quantum dispatcher API change. + Decoders *expected = nullptr; + if (!g_active_decoders.compare_exchange_strong(expected, &decoders_, + std::memory_order_acq_rel)) + throw std::runtime_error( + "qec_realtime_session: another HOST_CALL session is already active; " + "concurrent sessions are not supported"); try { - if (device_mode_) - capture_decoder_graphs(); allocate_ring_buffer(); populate_function_table(); - if (device_mode_) - start_device_loop(); // self-relaunching device-graph scheduler - else - start_host_loop(); + start_host_loop(); initialized_ = true; } catch (...) { - CUDA_QEC_WARN("qec_realtime_session::initialize: rolling back partial " - "initialization after exception"); finalize(); throw; } - - if (device_mode_) - CUDA_QEC_INFO("qec_realtime_session: initialized DEVICE mode " - "(num_decoders_with_graph={}, num_slots={}, slot_size={})", - num_decoders_with_graph_, num_slots_, slot_size_); - else - CUDA_QEC_INFO("qec_realtime_session: initialized HOST mode " - "(num_slots={}, slot_size={})", - num_slots_, slot_size_); } -//============================================================================== -// finalize() -//============================================================================== - void qec_realtime_session::finalize() { const bool was_initialized = initialized_; - if (was_initialized) - initialized_ = false; - // Note: we intentionally do NOT early-return on !was_initialized; the - // initialize() rollback path depends on running through the cleanup below. - - stop_loops(); - - if (device_mode_) { - // After stop_loops() the host monitor thread is joined and the persistent - // device kernel has been signalled to exit, but in-flight worker-stream - // graph launches submitted before the join can still be running. - // cudaDeviceSynchronize() drains all outstanding work, so the subsequent - // release_decode_graph() can't free buffers a still-running enqueue graph - // dereferences. finalize() is cold-path so the sync cost is irrelevant. - cudaDeviceSynchronize(); - - for (std::size_t i = 0; i < captured_graphs_.size(); ++i) { - if (captured_graphs_[i] && i < decoders_.size() && decoders_[i]) - decoders_[i]->release_decode_graph(captured_graphs_[i]); - } - } - captured_graphs_.clear(); - num_decoders_with_graph_ = 0; - - // Release the HOST-mode global if this session owns it (no-op otherwise). + initialized_ = false; + stop_host_loop(); + // Release the global only if this session owns it (no-op for a session that + // never acquired it, e.g. one rejected because another was already active). Decoders *self = &decoders_; g_active_decoders.compare_exchange_strong(self, nullptr, std::memory_order_acq_rel); - - // Free the function table with the allocator that matches the mode it was - // built with. - if (function_table_host_) { - if (device_mode_) - cudaFreeHost(function_table_host_); - else - std::free(function_table_host_); - function_table_host_ = nullptr; - function_table_dev_ = nullptr; - } - function_table_count_ = 0; - get_corrections_fn_id_ = 0; - reset_decoder_fn_id_ = 0; - - if (device_stats_dev_) { - cudaFree(device_stats_dev_); - device_stats_dev_ = nullptr; - } - - // Free the ring. DEVICE mode allocated pinned-mapped backings; HOST mode - // allocated plain host memory with _dev aliasing _host (free _host only). - auto free_ring_u64 = [&](volatile std::uint64_t *&host, - volatile std::uint64_t *&dev) { - if (host) { - if (device_mode_) - cudaFreeHost(const_cast(host)); - else - std::free(const_cast(host)); - } - host = nullptr; - dev = nullptr; - }; - auto free_ring_u8 = [&](std::uint8_t *&host, std::uint8_t *&dev) { - if (host) { - if (device_mode_) - cudaFreeHost(host); - else - std::free(host); - } - host = nullptr; - dev = nullptr; - }; - free_ring_u64(tx_flags_host_, tx_flags_dev_); - free_ring_u64(rx_flags_host_, rx_flags_dev_); - free_ring_u8(tx_data_host_, tx_data_dev_); - free_ring_u8(rx_data_host_, rx_data_dev_); - - if (shutdown_flag_host_) { - cudaFreeHost(shutdown_flag_host_); - shutdown_flag_host_ = nullptr; - shutdown_flag_dev_ = nullptr; - } - + function_table_.clear(); + rx_flags_.clear(); + tx_flags_.clear(); + rx_data_.clear(); + tx_data_.clear(); std::memset(&ringbuffer_, 0, sizeof(ringbuffer_)); - std::memset(&host_table_, 0, sizeof(host_table_)); - std::memset(&host_config_, 0, sizeof(host_config_)); - host_engine_ = nullptr; - + std::memset(&host_ctx_, 0, sizeof(host_ctx_)); if (was_initialized) - CUDA_QEC_INFO("qec_realtime_session: finalized"); -} - -//============================================================================== -// capture_decoder_graphs() [DEVICE mode] -//============================================================================== - -void qec_realtime_session::capture_decoder_graphs() { - captured_graphs_.assign(decoders_.size(), nullptr); - num_decoders_with_graph_ = 0; - - // kMaxDispatchedDecoders sizes the device-side g_decoder_state_table[]. - if (decoders_.size() > cudaq::qec::decoding::rpc::kMaxDispatchedDecoders) - throw std::runtime_error( - "qec_realtime_session::initialize: requested " + - std::to_string(decoders_.size()) + - " decoders but the realtime dispatch supports at most " + - std::to_string(cudaq::qec::decoding::rpc::kMaxDispatchedDecoders) + - " (kMaxDispatchedDecoders)."); - - for (std::size_t i = 0; i < decoders_.size(); ++i) { - auto *dec = decoders_[i].get(); - if (!dec) - continue; - if (!dec->supports_graph_dispatch()) - throw std::runtime_error( - "qec_realtime_session::initialize: decoder " + std::to_string(i) + - " does not support graph dispatch in DEVICE mode."); - - apply_decoder_cuda_device(dec); - // reserved_sms = 0 is intentional for the inproc_rpc desktop / CI path. - void *raw = dec->capture_decode_graph(/*reserved_sms=*/0); - if (!raw) - throw std::runtime_error("qec_realtime_session::initialize: decoder " + - std::to_string(i) + - " returned null from capture_decode_graph()"); - captured_graphs_[i] = raw; - - auto *gres = static_cast(raw); - if (!gres->graph_exec || !gres->function_id) - throw std::runtime_error( - "qec_realtime_session::initialize: decoder " + std::to_string(i) + - " produced incomplete graph_resources (graph_exec / function_id)"); - - // All N enqueue_syndromes graphs share a single canonical function_id; the - // host monitor disambiguates per-decoder via routing_key == decoder_id. - if (gres->function_id != - cudaq::qec::decoding::rpc::kEnqueueSyndromesFunctionId) - throw std::runtime_error( - "qec_realtime_session::initialize: decoder " + std::to_string(i) + - " published a non-canonical enqueue function_id"); - - ++num_decoders_with_graph_; - } - - if (num_decoders_with_graph_ == 0) - throw std::runtime_error( - "qec_realtime_session::initialize: no decoders to capture graphs for"); + CUDA_QEC_INFO("qec_realtime_session: finalized HOST_CALL session"); } -//============================================================================== -// allocate_ring_buffer() [branches on device_mode_] -//============================================================================== - void qec_realtime_session::allocate_ring_buffer() { namespace rpc = cudaq::qec::decoding::rpc; - using cudaq::realtime::RPCHeader; - using cudaq::realtime::RPCResponse; - - // Slot size: largest body across the RPC trio, over all served decoders. - std::size_t max_measurements = 0; + std::size_t max_syndromes = 0; std::size_t max_observables = 0; - for (std::size_t i = 0; i < decoders_.size(); ++i) { - auto *dec = decoders_[i].get(); - if (!dec) - continue; - // DEVICE mode only sizes for decoders that captured a graph. - if (device_mode_ && !captured_graphs_[i]) + for (auto &decoder : decoders_) { + if (!decoder) continue; - max_measurements = - std::max(max_measurements, dec->get_num_msyn_per_decode()); + max_syndromes = std::max(max_syndromes, + decoder->get_num_msyn_per_decode()); max_observables = - std::max(max_observables, dec->get_num_observables()); - } - - const std::size_t enqueue_req = sizeof(RPCHeader) + - sizeof(rpc::EnqueueRequestPayload) + - rpc::bit_packed_bytes(max_measurements); - const std::size_t get_req = - sizeof(RPCHeader) + sizeof(rpc::GetCorrectionsRequestPayload); - const std::size_t reset_req = - sizeof(RPCHeader) + sizeof(rpc::ResetRequestPayload); - const std::size_t enqueue_resp = sizeof(RPCResponse); - const std::size_t get_resp = - sizeof(RPCResponse) + rpc::bit_packed_bytes(max_observables); - const std::size_t reset_resp = sizeof(RPCResponse); - - slot_size_ = std::max({enqueue_req, get_req, reset_req, enqueue_resp, - get_resp, reset_resp, std::size_t{64}}); - - // Round the slot stride up to an alignment boundary so every slot -- and thus - // every RPCHeader/RPCResponse placed at slot offset i*slot_size_ -- starts - // aligned. write_response() (and the producer) perform a 4-byte __atomic RMW - // on the header/response `magic` field at slot offset 0; with an unaligned - // stride (e.g. slot_size_ == 75 at distance 5) that atomic lands on an - // unaligned address, which the CPU splits into a bus-locked, cache-line- - // crossing access -- fatal SIGBUS on hosts with split-lock detection enabled - // (common on cloud / CI runners, tolerated silently elsewhere). A distance-3 - // config happens to floor at the aligned 64-byte minimum and so never hit - // this. DEVICE mode additionally wants a 256-byte stride for deterministic - // GPU-visible slot addressing; HOST mode only needs the atomics aligned. - { - constexpr std::size_t kDeviceSlotAlignment = 256; - constexpr std::size_t kHostSlotAlignment = 16; - const std::size_t alignment = - device_mode_ ? kDeviceSlotAlignment : kHostSlotAlignment; - slot_size_ = (slot_size_ + (alignment - 1)) & ~(alignment - 1); + std::max(max_observables, decoder->get_num_observables()); } - if (device_mode_) { - auto alloc_u64 = [&](volatile std::uint64_t *&host, - volatile std::uint64_t *&dev, const char *what) { - void *h = nullptr; - void *d = nullptr; - if (!allocate_pinned_mapped(num_slots_ * sizeof(std::uint64_t), &h, &d)) - throw std::runtime_error( - std::string( - "qec_realtime_session::initialize: failed to allocate ") + - what); - host = static_cast(h); - dev = static_cast(d); - }; - auto alloc_u8 = [&](std::uint8_t *&host, std::uint8_t *&dev, - const char *what) { - void *h = nullptr; - void *d = nullptr; - if (!allocate_pinned_mapped(num_slots_ * slot_size_, &h, &d)) - throw std::runtime_error( - std::string( - "qec_realtime_session::initialize: failed to allocate ") + - what); - host = static_cast(h); - dev = static_cast(d); - }; - alloc_u64(rx_flags_host_, rx_flags_dev_, "rx_flags"); - alloc_u64(tx_flags_host_, tx_flags_dev_, "tx_flags"); - alloc_u8(rx_data_host_, rx_data_dev_, "RX ring data"); - alloc_u8(tx_data_host_, tx_data_dev_, "TX ring data"); - - { - void *h = nullptr; - void *d = nullptr; - if (!allocate_pinned_mapped(sizeof(int), &h, &d)) - throw std::runtime_error("qec_realtime_session::initialize: failed to " - "allocate shutdown flag"); - shutdown_flag_host_ = static_cast(h); - *shutdown_flag_host_ = 0; - shutdown_flag_dev_ = static_cast(d); - } - } else { - // HOST mode: plain host memory; the device-visible pointers alias the host - // backings (no GPU required at runtime). The host loop reads only the - // *_host views; the producer's address-as-flag publish uses rx_data_dev() - // (== rx_data_host_ here), which the host loop dereferences as host memory. - auto alloc_u64 = [&](volatile std::uint64_t *&host, - volatile std::uint64_t *&dev, const char *what) { - void *p = std::calloc(num_slots_, sizeof(std::uint64_t)); - if (!p) - throw std::runtime_error( - std::string( - "qec_realtime_session::initialize: failed to allocate ") + - what); - host = static_cast(p); - dev = host; - }; - auto alloc_u8 = [&](std::uint8_t *&host, std::uint8_t *&dev, - const char *what) { - void *p = std::calloc(num_slots_, slot_size_); - if (!p) - throw std::runtime_error( - std::string( - "qec_realtime_session::initialize: failed to allocate ") + - what); - host = static_cast(p); - dev = host; - }; - alloc_u64(rx_flags_host_, rx_flags_dev_, "rx_flags"); - alloc_u64(tx_flags_host_, tx_flags_dev_, "tx_flags"); - alloc_u8(rx_data_host_, rx_data_dev_, "RX ring data"); - alloc_u8(tx_data_host_, tx_data_dev_, "TX ring data"); - } + const std::size_t enqueue_bytes = + sizeof(cudaq::realtime::RPCHeader) + + rpc::align_to_8(sizeof(rpc::EnqueueRequestPayload) + + rpc::bit_packed_bytes(max_syndromes)); + const std::size_t get_request_bytes = + sizeof(cudaq::realtime::RPCHeader) + + sizeof(rpc::GetCorrectionsRequestPayload); + const std::size_t get_response_bytes = + sizeof(cudaq::realtime::RPCResponse) + + rpc::align_to_8(rpc::bit_packed_bytes(max_observables)); + const std::size_t reset_bytes = + sizeof(cudaq::realtime::RPCHeader) + sizeof(rpc::ResetRequestPayload); + slot_size_ = std::max({enqueue_bytes, get_request_bytes, get_response_bytes, + reset_bytes, std::size_t{64}}); + + rx_flags_.assign(num_slots_, 0); + tx_flags_.assign(num_slots_, 0); + rx_data_.assign(num_slots_ * slot_size_, 0); + tx_data_.assign(num_slots_ * slot_size_, 0); std::memset(&ringbuffer_, 0, sizeof(ringbuffer_)); - ringbuffer_.rx_flags = rx_flags_dev_; - ringbuffer_.tx_flags = tx_flags_dev_; - ringbuffer_.rx_data = rx_data_dev_; - ringbuffer_.tx_data = tx_data_dev_; + ringbuffer_.rx_flags_host = rx_flags_.data(); + ringbuffer_.tx_flags_host = tx_flags_.data(); + ringbuffer_.rx_data_host = rx_data_.data(); + ringbuffer_.tx_data_host = tx_data_.data(); ringbuffer_.rx_stride_sz = slot_size_; ringbuffer_.tx_stride_sz = slot_size_; - ringbuffer_.rx_flags_host = rx_flags_host_; - ringbuffer_.tx_flags_host = tx_flags_host_; - ringbuffer_.rx_data_host = rx_data_host_; - ringbuffer_.tx_data_host = tx_data_host_; } -//============================================================================== -// populate_function_table() [branches on device_mode_] -//============================================================================== - void qec_realtime_session::populate_function_table() { namespace rpc = cudaq::qec::decoding::rpc; + function_table_.assign(3, cudaq_function_entry_t{}); - if (!device_mode_) { - // HOST mode: 3 HOST_CALL entries (enqueue, get_corrections, reset). Plain - // host allocation -- host_fn pointers are host code addresses; _dev aliases - // _host. decoder_id routing happens inside each handler via the payload. - function_table_count_ = 3; - void *p = - std::calloc(function_table_count_, sizeof(cudaq_function_entry_t)); - if (!p) - throw std::runtime_error("qec_realtime_session::initialize: failed to " - "allocate function table"); - function_table_host_ = static_cast(p); - function_table_dev_ = function_table_host_; - - function_table_host_[0].handler.host_fn = enqueue_syndromes_host; - function_table_host_[0].function_id = rpc::kEnqueueSyndromesFunctionId; - function_table_host_[0].dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; - - function_table_host_[1].handler.host_fn = get_corrections_host; - function_table_host_[1].function_id = rpc::kGetCorrectionsFunctionId; - function_table_host_[1].dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; - - function_table_host_[2].handler.host_fn = reset_decoder_host; - function_table_host_[2].function_id = rpc::kResetDecoderFunctionId; - function_table_host_[2].dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; - - get_corrections_fn_id_ = rpc::kGetCorrectionsFunctionId; - reset_decoder_fn_id_ = rpc::kResetDecoderFunctionId; - return; - } - - // DEVICE mode: 3 DEVICE_CALL entries (enqueue_syndromes accumulate, - // get_corrections, reset_decoder), all serviced by the self-relaunching - // device-graph scheduler. enqueue_syndromes is an accumulate handler (NOT a - // GRAPH_LAUNCH): it appends the round's syndromes into the registered - // GpuDecoderState and returns CUDAQ_DISPATCH_STATUS_TRIGGER_GRAPH when a full - // window has accumulated, which tells the scheduler to fire the per-decoder - // device-launchable decode graph fire-and-forget. All three handlers live - // in the cudevice archive and are resolved by name via dlsym so a .so that - // did not link that archive can still load. Pinned-mapped so the scheduler - // kernel reads the same backing. - function_table_count_ = 3; - - void *h = nullptr; - void *d = nullptr; - if (!allocate_pinned_mapped( - function_table_count_ * sizeof(cudaq_function_entry_t), &h, &d)) - throw std::runtime_error( - "qec_realtime_session::initialize: failed to allocate function table"); - function_table_host_ = static_cast(h); - function_table_dev_ = static_cast(d); - - // Resolve a DEVICE_CALL populate shim by name, invoke it on the target entry, - // stamp the function_id/routing_key, and validate it produced a real - // DEVICE_CALL handler. routing_key is unused by the scheduler (single - // decoder) but kept 0 for forward compatibility with per-decoder routing. - auto populate_device_call = [&](std::size_t slot, const char *symbol, - std::uint32_t function_id) { - auto shim = resolve_populate_shim(symbol); - if (!shim) - throw std::runtime_error( - std::string("qec_realtime_session::initialize: ") + symbol + - " not found via dlsym(RTLD_DEFAULT, ...). The final binary must " - "link libcudaq-qec-realtime-cudevice.a (or the static parts of " - "decoder_rpc_dispatch.cu via qec_realtime_app_link_options())."); - shim(&function_table_host_[slot]); - function_table_host_[slot].function_id = function_id; - function_table_host_[slot].routing_key = 0; - if (function_table_host_[slot].dispatch_mode != - CUDAQ_DISPATCH_DEVICE_CALL || - !function_table_host_[slot].handler.device_fn_ptr) - throw std::runtime_error( - std::string("qec_realtime_session::initialize: ") + symbol + - " did not produce a valid DEVICE_CALL entry (plugin bug)"); - }; - - // [0] enqueue_syndromes accumulate. - populate_device_call( - 0, "cudaqx_qec_realtime_dispatch_populate_enqueue_syndromes_device_entry", - rpc::kEnqueueSyndromesFunctionId); - // [1] get_corrections. - get_corrections_fn_id_ = rpc::kGetCorrectionsFunctionId; - populate_device_call( - 1, "cudaqx_qec_realtime_dispatch_populate_get_corrections_device_entry", - get_corrections_fn_id_); - // [2] reset_decoder. - reset_decoder_fn_id_ = rpc::kResetDecoderFunctionId; - populate_device_call( - 2, "cudaqx_qec_realtime_dispatch_populate_reset_decoder_device_entry", - reset_decoder_fn_id_); -} - -//============================================================================== -// start_device_loop() [DEVICE mode] -//============================================================================== - -void qec_realtime_session::start_device_loop() { - // The scheduler fires a single per-decoder decode graph fire-and-forget when - // the enqueue accumulate handler signals a full window. The public host - // dispatcher exposes one triggered graph per scheduler, so DEVICE mode is - // scoped to a single graph-dispatch decoder (matching the prior single- - // mailbox-bank limitation). A multi-decoder scheduler would need a - // function-id/routing-key -> triggered-graph map inside the dispatch kernel. - if (num_decoders_with_graph_ != 1) - throw std::runtime_error( - "qec_realtime_session::initialize: the device-graph scheduler supports " - "exactly one graph-dispatch decoder per session (got " + - std::to_string(num_decoders_with_graph_) + ")."); - - // The lone captured decoder's device-launchable decode graph. - cudaGraphExec_t decode_graph_exec = nullptr; - for (std::size_t i = 0; i < captured_graphs_.size(); ++i) { - if (!captured_graphs_[i]) - continue; - auto *gres = static_cast( - captured_graphs_[i]); - decode_graph_exec = gres->graph_exec; - break; - } - if (!decode_graph_exec) - throw std::runtime_error( - "qec_realtime_session::initialize: no captured decode graph for the " - "scheduler to trigger"); - - if (cudaMalloc(&device_stats_dev_, sizeof(std::uint64_t)) != cudaSuccess || - cudaMemset(device_stats_dev_, 0, sizeof(std::uint64_t)) != cudaSuccess) - throw std::runtime_error( - "qec_realtime_session::initialize: device_stats_dev allocation failed"); + function_table_[0].handler.host_fn = enqueue_syndromes_host; + function_table_[0].function_id = rpc::kEnqueueSyndromesFunctionId; + function_table_[0].dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; - if (cudaStreamCreate(&scheduler_stream_) != cudaSuccess) - throw std::runtime_error( - "qec_realtime_session::initialize: cudaStreamCreate for the scheduler " - "stream failed"); - - // Bind the graph dispatch API from the host executable's device-link unit - // (NOT from any copy this .so might carry) so the dispatch kernel and the - // decoder's DEVICE_CALL handlers share one CUDA module -- see - // resolve_dispatch_graph_api(). The destroy fn is stashed for stop_loops(). - auto create_fn = resolve_dispatch_graph_api( - "cudaq_create_dispatch_graph_regular"); - auto launch_fn = resolve_dispatch_graph_api( - "cudaq_launch_dispatch_graph"); - destroy_dispatch_graph_fn_ = - resolve_dispatch_graph_api( - "cudaq_destroy_dispatch_graph"); - - // The scheduler kernel itself is lightweight (poll + parse + DEVICE_CALL + - // fire-and-forget); the heavy cooperative decode lives in the triggered - // graph, so a single-block scheduler is sufficient. - cudaError_t err = - create_fn(rx_flags_dev_, tx_flags_dev_, rx_data_dev_, tx_data_dev_, - slot_size_, slot_size_, function_table_dev_, - static_cast(function_table_count_), - /*graph_io_ctx=*/nullptr, shutdown_flag_dev_, device_stats_dev_, - num_slots_, /*num_blocks=*/1, /*threads_per_block=*/64, - decode_graph_exec, scheduler_stream_, &scheduler_ctx_); - if (err != cudaSuccess) - throw std::runtime_error( - std::string("qec_realtime_session::initialize: " - "cudaq_create_dispatch_graph_regular failed: ") + - cudaGetErrorString(err)); + function_table_[1].handler.host_fn = get_corrections_host; + function_table_[1].function_id = rpc::kGetCorrectionsFunctionId; + function_table_[1].dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; - err = launch_fn(scheduler_ctx_, scheduler_stream_); - if (err != cudaSuccess) - throw std::runtime_error( - std::string("qec_realtime_session::initialize: " - "cudaq_launch_dispatch_graph failed: ") + - cudaGetErrorString(err)); + function_table_[2].handler.host_fn = reset_decoder_host; + function_table_[2].function_id = rpc::kResetDecoderFunctionId; + function_table_[2].dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; } -//============================================================================== -// start_host_loop() [HOST mode only] -//============================================================================== - void qec_realtime_session::start_host_loop() { - // HOST mode only: inline HOST_CALL handlers, no graph worker pool. DEVICE - // mode uses the self-relaunching device-graph scheduler launched in - // start_device_loop() (see initialize()'s dispatch), so this is reached only - // when device_mode_ is false. Every table entry is HOST_CALL, so a - // GRAPH_LAUNCH engine would build no workers -- we drive the ring loop with a - // NULL engine (the loop runs the HOST_CALL handlers inline). - std::memset(&host_config_, 0, sizeof(host_config_)); - host_config_.num_slots = static_cast(num_slots_); - host_config_.slot_size = static_cast(slot_size_); - host_config_.dispatch_path = CUDAQ_DISPATCH_PATH_HOST; - host_config_.dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; - host_config_.skip_tx_markers = 1; - // Strict-FIFO: the host loop is the sole consumer and its current_slot - // already advances monotonically; with shared-ring scanning OFF it simply - // waits at current_slot for the next frame. The rpc_producer advances its - // slot monotonically too (see producer_cursor()), so producer and consumer - // walk the ring in lockstep -- no scan needed. - host_config_.shared_ring_mode = 0; - host_table_.entries = function_table_host_; - host_table_.count = static_cast(function_table_count_); - host_engine_ = nullptr; shutdown_flag_ = 0; - - host_loop_thread_ = std::thread([this]() { - cudaq_host_ring_dispatch_loop(&ringbuffer_, &host_table_, &host_config_, - /*engine=*/nullptr, &shutdown_flag_, - &host_stats_counter_); - }); + std::memset(&host_ctx_, 0, sizeof(host_ctx_)); + host_ctx_.ringbuffer = ringbuffer_; + host_ctx_.config.num_slots = static_cast(num_slots_); + host_ctx_.config.slot_size = static_cast(slot_size_); + host_ctx_.config.dispatch_path = CUDAQ_DISPATCH_PATH_HOST; + host_ctx_.config.dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; + host_ctx_.config.skip_tx_markers = 1; + host_ctx_.function_table.entries = function_table_.data(); + host_ctx_.function_table.count = + static_cast(function_table_.size()); + host_ctx_.shutdown_flag = &shutdown_flag_; + host_ctx_.stats_counter = &host_stats_counter_; + host_ctx_.skip_stream_sweep = true; + + host_loop_thread_ = + std::thread([this]() { cudaq_host_dispatcher_loop(&host_ctx_); }); } -//============================================================================== -// stop_loops() -//============================================================================== - -void qec_realtime_session::stop_loops() { - // Signal shutdown to whichever flag the active host loop polls (and, in - // DEVICE mode, the persistent device kernel which shares the pinned flag). - if (device_mode_) { - if (shutdown_flag_host_) { - __atomic_store_n(shutdown_flag_host_, 1, __ATOMIC_RELEASE); - __sync_synchronize(); - } - } else { +void qec_realtime_session::stop_host_loop() { + if (host_loop_thread_.joinable()) { __atomic_store_n(&shutdown_flag_, 1, __ATOMIC_RELEASE); __sync_synchronize(); - } - - if (host_loop_thread_.joinable()) host_loop_thread_.join(); - - // Drain and destroy the device-graph scheduler (DEVICE mode). The shutdown - // flag was set above; the scheduler observes it, stops tail-self-relaunching, - // and the in-flight self-relaunch chain drains. cudaStreamSynchronize waits - // for that to finish before we destroy the graph context. - if (scheduler_ctx_) { - if (scheduler_stream_) - cudaStreamSynchronize(scheduler_stream_); - if (destroy_dispatch_graph_fn_) - destroy_dispatch_graph_fn_(scheduler_ctx_); - scheduler_ctx_ = nullptr; - } - if (scheduler_stream_) { - cudaStreamDestroy(scheduler_stream_); - scheduler_stream_ = nullptr; - } - - // Legacy DEVICE_LOOP teardown (unused by the scheduler; both are null in - // scheduler mode, so these are harmless no-ops). - if (device_dispatcher_) { - cudaq_dispatcher_stop(device_dispatcher_); - cudaq_dispatcher_destroy(device_dispatcher_); - device_dispatcher_ = nullptr; - } - if (device_manager_) { - cudaq_dispatch_manager_destroy(device_manager_); - device_manager_ = nullptr; - } - - // The engine owns the worker streams, idle mask, inflight-slot tags, and - // GraphIOContext array; destroy it after the driving loop has stopped. - if (host_engine_) { - cudaq_graph_launch_engine_destroy(host_engine_); - host_engine_ = nullptr; } } diff --git a/libs/qec/lib/realtime/qec_realtime_session.h b/libs/qec/lib/realtime/qec_realtime_session.h index fef32a6bc..c28431fb2 100644 --- a/libs/qec/lib/realtime/qec_realtime_session.h +++ b/libs/qec/lib/realtime/qec_realtime_session.h @@ -10,19 +10,11 @@ #ifdef CUDAQ_REALTIME_ROOT -// Defines CUDA_VERSION, which gates the graph-based dispatch API (the -// self-relaunching scheduler: cudaq_dispatch_graph_context, -// cudaq_create_dispatch_graph_regular, ...) inside cudaq_realtime.h. This TU -// is compiled by the host C++ compiler (no __CUDACC__), so without this the -// graph API would be preprocessed out and the scheduler members below would -// not name a type. -#include - #include "cudaq/qec/decoder.h" #include "cudaq/realtime/daemon/dispatcher/cudaq_realtime.h" -#include "cudaq/realtime/daemon/dispatcher/dispatch_kernel_launch.h" -#include "cudaq/realtime/daemon/dispatcher/graph_launch_engine.h" +#include "cudaq/realtime/daemon/dispatcher/host_dispatcher.h" +#include #include #include #include @@ -30,232 +22,46 @@ namespace cudaq::qec::realtime { -/// @brief Per-process realtime decoding session, dual-mode. -/// -/// A session is **homogeneous**: at `initialize()` it inspects the decoders and -/// runs one of two dispatch modes, chosen by -/// `decoder::supports_graph_dispatch()`: -/// -/// - DEVICE mode (every decoder supports graph dispatch -- e.g. the Relay BP -/// GPU decoder). A single self-relaunching device-graph scheduler (see -/// `start_device_loop()`) services all three RPCs as `DEVICE_CALL` -/// handlers: `enqueue_syndromes` accumulates the round's syndromes into the -/// decoder's registered GpuDecoderState and returns -/// `CUDAQ_DISPATCH_STATUS_TRIGGER_GRAPH` when a full window is ready, which -/// makes the scheduler fire the decoder's device-launchable cooperative -/// decode graph fire-and-forget; `get_corrections` / `reset_decoder` run -/// inline on device. The scheduler tail-self-relaunches via -/// `cudaGetCurrentGraphExec()` so the 120 fire-and-forget launches per -/// parent-graph execution budget resets each relaunch. Scoped to one -/// graph-dispatch decoder per session. Requires a non-null -/// `device_launch_fn`. -/// -/// - HOST mode (no decoder supports graph dispatch -- e.g. PyMatching, a CPU -/// decoder). All three RPCs (enqueue_syndromes, get_corrections, -/// reset_decoder) are `CUDAQ_DISPATCH_HOST_CALL` handlers invoked inline by -/// the CPU HOST_LOOP using the two-ring callback ABI -/// `cudaq_host_rpc_fn_t(const void *rx, void *tx, size_t)`. The ring is -/// plain host memory (no GPU required at runtime); the device-visible -/// pointers alias the host backings so `rpc_producer` is mode-agnostic. -/// -/// A *mixed* session (some decoders support graph dispatch and some do not) is -/// rejected: the host loop resolves a slot to a function table entry by -/// `function_id` alone, so a `GRAPH_LAUNCH` enqueue and a `HOST_CALL` enqueue -/// sharing `kEnqueueSyndromesFunctionId` would collide. One decoder per -/// session is the supported (and tested) configuration; homogeneous multi-CPU -/// sessions also work. See the throw in `classify_mode()`. -/// -/// Ring layout (both modes): producer writes RPCHeader + payload into the RX -/// backing, the dispatcher writes RPCResponse + result into the TX backing. -/// RX and TX are separate physical allocations. -/// -/// Constructed with a reference to a vector of realized decoder instances -- -/// the same vector held by `realtime_decoding.cpp::g_decoders` in the -/// production path, or a one-element vector in a unit test. The session keeps -/// a non-owning reference so it can call `release_decode_graph()` (DEVICE mode) -/// on each captured graph at finalize time. -/// -/// The class is marked `default`-visible so its constructor / destructor / -/// `initialize` / `finalize` symbols cross the `cudaq-qec-realtime-decoding` -/// shared-library boundary (the library is built with `-fvisibility=hidden`). class __attribute__((visibility("default"))) qec_realtime_session { public: - /// @brief Construct a session over the given realized decoders. - /// @param decoders Reference must outlive this session. The session calls - /// `supports_graph_dispatch()` on each non-null entry at - /// `initialize()` time to choose its dispatch mode, and (in - /// DEVICE mode) `capture_decode_graph()` / - /// `release_decode_graph()`. - /// @param device_launch_fn Function pointer passed to - /// `cudaq_dispatcher_set_launch_fn` in DEVICE mode. Typically - /// `&cudaq_launch_dispatch_kernel_regular` from libcudaq- - /// realtime-dispatch. Passed in (rather than referenced - /// directly) so this shared library stays free of references - /// to symbols that live only in static archives linked by the - /// final executable. May be null for a HOST-mode (CPU - /// decoder) session; `initialize()` throws if a DEVICE-mode - /// decoder set is given without it. explicit qec_realtime_session( - std::vector> &decoders, - cudaq_dispatch_launch_fn_t device_launch_fn = nullptr); - + std::vector> &decoders); ~qec_realtime_session(); - qec_realtime_session(const qec_realtime_session &) = delete; - qec_realtime_session &operator=(const qec_realtime_session &) = delete; - qec_realtime_session(qec_realtime_session &&) = delete; - qec_realtime_session &operator=(qec_realtime_session &&) = delete; - - /// @brief Bring up the ring + dispatcher(s) for the selected mode. - /// Idempotent: a second call is a no-op. Throws `std::runtime_error` on any - /// failure (mixed decoder set, DEVICE mode without device_launch_fn, decoder - /// lacks graph dispatch in DEVICE mode, CUDA allocation failure, plugin - /// failed to populate device entries, a second concurrent HOST-mode session, - /// ...). void initialize(); - - /// @brief Tear down dispatcher(s), release captured graphs (DEVICE mode), - /// free ring. Idempotent. Safe to call from a destructor. In DEVICE mode - /// must be called BEFORE the decoders vector is cleared. void finalize(); - - /// @brief True if `initialize()` has completed and `finalize()` has not. bool initialized() const { return initialized_; } - /// @brief True if this session runs in DEVICE (GPU graph-dispatch) mode. - /// Only meaningful after `initialize()`. - bool device_mode() const { return device_mode_; } - - // ---- Accessors used by rpc_producer.cpp (and by tests). ---------------- - // In HOST mode the `_dev` pointers alias the `_host` backings (host memory), - // so the producer's address-as-flag publish works unchanged in both modes. - - volatile std::uint64_t *rx_flags_host() const { return rx_flags_host_; } - volatile std::uint64_t *tx_flags_host() const { return tx_flags_host_; } - std::uint8_t *rx_data_host() const { return rx_data_host_; } - std::uint8_t *rx_data_dev() const { return rx_data_dev_; } - std::uint8_t *tx_data_host() const { return tx_data_host_; } - std::uint8_t *tx_data_dev() const { return tx_data_dev_; } - + volatile std::uint64_t *rx_flags_host() { return rx_flags_.data(); } + volatile std::uint64_t *tx_flags_host() { return tx_flags_.data(); } + std::uint8_t *rx_data_host() { return rx_data_.data(); } + std::uint8_t *tx_data_host() { return tx_data_.data(); } std::size_t num_slots() const { return num_slots_; } std::size_t slot_size() const { return slot_size_; } - /// @brief Monotonic producer cursor (rpc_producer ring discipline). - /// The single serialized producer advances this each RPC so it walks the - /// ring in lockstep with the strict-FIFO consumer (device-graph scheduler or - /// host loop, both with shared-ring scanning OFF) instead of reusing slot 0. - /// Reset to 0 by initialize(). Single-producer today; a future multi- - /// producer design would make the advance an atomic fetch-add. - std::size_t producer_cursor() const { return producer_cursor_; } - void set_producer_cursor(std::size_t slot) { producer_cursor_ = slot; } - - /// @brief (DEVICE mode) Number of decoders that captured a CUDA graph. - std::size_t num_decoders_with_graph() const { - return num_decoders_with_graph_; - } - private: - // Inspect decoders_ and set device_mode_. Throws on an empty set, a mixed - // (graph + non-graph) set, or a DEVICE-mode set without device_launch_fn_. - void classify_mode(); - - // ---- DEVICE-mode internals ---- - void capture_decoder_graphs(); - void start_device_loop(); - - // ---- shared internals (branch on device_mode_) ---- - // allocate_ring_buffer() computes slot_size_, allocates rx/tx flags + data - // (pinned-mapped in DEVICE mode, host memory in HOST mode), and fully - // populates ringbuffer_. void allocate_ring_buffer(); - // populate_function_table() builds the shared function table: N GRAPH_LAUNCH - // + 2 DEVICE_CALL entries in DEVICE mode; 3 HOST_CALL entries in HOST mode. void populate_function_table(); - // start_host_loop() launches the CPU HOST_LOOP thread. In DEVICE mode it - // wires the per-decoder graph workers + GraphIOContext mailbox; in HOST mode - // it runs the inline HOST_CALL handlers (no worker pool). void start_host_loop(); + void stop_host_loop(); - // Signal shutdown, join host thread, stop device dispatcher (if any), free - // worker streams + per-worker storage. - void stop_loops(); - - // ---- References / external state ---- std::vector> &decoders_; - cudaq_dispatch_launch_fn_t device_launch_fn_ = nullptr; - // ---- Lifetime / mode ---- bool initialized_ = false; - bool device_mode_ = false; - - // ---- Ring buffer (raw pointers; _dev aliases _host in HOST mode) ---- - static constexpr std::size_t kDefaultNumSlots = 8; - std::size_t num_slots_ = kDefaultNumSlots; + std::size_t num_slots_ = 8; std::size_t slot_size_ = 0; - // Monotonic producer ring cursor (see producer_cursor()). Reset in - // initialize(). - std::size_t producer_cursor_ = 0; - volatile std::uint64_t *rx_flags_host_ = nullptr; - volatile std::uint64_t *rx_flags_dev_ = nullptr; - volatile std::uint64_t *tx_flags_host_ = nullptr; - volatile std::uint64_t *tx_flags_dev_ = nullptr; - std::uint8_t *rx_data_host_ = nullptr; - std::uint8_t *rx_data_dev_ = nullptr; - std::uint8_t *tx_data_host_ = nullptr; - std::uint8_t *tx_data_dev_ = nullptr; - cudaq_ringbuffer_t ringbuffer_{}; - - // ---- Function table ---- - // DEVICE mode: pinned-mapped (host + device same UVA). HOST mode: plain host - // allocation (host_fn pointers are host code addresses); _dev aliases _host. - std::size_t function_table_count_ = 0; - cudaq_function_entry_t *function_table_host_ = nullptr; - cudaq_function_entry_t *function_table_dev_ = nullptr; - std::uint32_t get_corrections_fn_id_ = 0; - std::uint32_t reset_decoder_fn_id_ = 0; - - // ---- DEVICE-mode scheduler wiring ---- - // A single self-relaunching device-graph scheduler replaces the legacy - // HOST_LOOP(graph-worker) + DEVICE_LOOP(get/reset) pair. The scheduler - // runs all three RPCs as DEVICE_CALL handlers and fires the - // per-decoder device-launchable decode graph fire-and-forget when the - // enqueue accumulate handler signals a full window - // (CUDAQ_DISPATCH_STATUS_TRIGGER_GRAPH). device_manager_/device_dispatcher_ - // are retained (unused in scheduler mode) to keep the legacy DEVICE_LOOP - // teardown in stop_loops() a harmless no-op. - cudaq_dispatch_manager_t *device_manager_ = nullptr; - cudaq_dispatcher_t *device_dispatcher_ = nullptr; - std::uint64_t *device_stats_dev_ = nullptr; - cudaq_dispatch_graph_context *scheduler_ctx_ = nullptr; - cudaStream_t scheduler_stream_ = nullptr; - // cudaq_destroy_dispatch_graph, resolved by start_device_loop() via - // dlsym(RTLD_DEFAULT, ...) from the host executable's absorbed - // libcudaq-realtime-dispatch.a (same image as the create/launch fns), so - // stop_loops() tears down the graph context with the copy that created it. - using destroy_dispatch_graph_fn_t = - cudaError_t (*)(cudaq_dispatch_graph_context *); - destroy_dispatch_graph_fn_t destroy_dispatch_graph_fn_ = nullptr; - // Pinned-mapped shutdown flag polled by the scheduler graph (DEVICE mode). - int *shutdown_flag_host_ = nullptr; - int *shutdown_flag_dev_ = nullptr; - // ---- HOST_LOOP wiring (both modes) ---- - // The GRAPH_LAUNCH engine (DEVICE mode) owns the worker streams, idle mask, - // inflight-slot tags, and per-worker GraphIOContext array. It is built from - // the GRAPH_LAUNCH entries of `function_table_host_` and is NULL in HOST mode - // (all-HOST_CALL table -> no graph workers). - cudaq_function_table_t host_table_{}; - cudaq_dispatcher_config_t host_config_{}; - cudaq_graph_launch_engine_t *host_engine_ = nullptr; - std::thread host_loop_thread_; - std::uint64_t host_stats_counter_ = 0; - // Plain (non-pinned) shutdown flag for HOST mode (no device kernel shares - // it). + std::vector rx_flags_; + std::vector tx_flags_; + std::vector rx_data_; + std::vector tx_data_; int shutdown_flag_ = 0; + std::uint64_t host_stats_counter_ = 0; - // ---- Graph state (DEVICE mode only) ---- - std::vector captured_graphs_; - std::size_t num_decoders_with_graph_ = 0; + std::vector function_table_; + cudaq_ringbuffer_t ringbuffer_{}; + cudaq_host_dispatch_loop_ctx_t host_ctx_{}; + std::thread host_loop_thread_; }; } // namespace cudaq::qec::realtime diff --git a/libs/qec/lib/realtime/quantinuum/CMakeLists.txt b/libs/qec/lib/realtime/quantinuum/CMakeLists.txt index 889bcb456..c56b44ebe 100644 --- a/libs/qec/lib/realtime/quantinuum/CMakeLists.txt +++ b/libs/qec/lib/realtime/quantinuum/CMakeLists.txt @@ -25,10 +25,7 @@ target_include_directories(cudaq-qec-realtime-decoding-quantinuum $ ) -# PRIVATE, not PUBLIC: PUBLIC propagates --exclude-libs,ALL to every consumer, -# including test executables that absorb libcudaq-realtime-dispatch.a and -# dlsym() its CUDAQ_REALTIME_DISPATCH_API symbols at runtime. -target_link_options(cudaq-qec-realtime-decoding-quantinuum PRIVATE +target_link_options(cudaq-qec-realtime-decoding-quantinuum PUBLIC $<$:-Wl,--exclude-libs,ALL> ) diff --git a/libs/qec/lib/realtime/realtime_decoding.cpp b/libs/qec/lib/realtime/realtime_decoding.cpp index 22ba4b1e4..6b290de72 100644 --- a/libs/qec/lib/realtime/realtime_decoding.cpp +++ b/libs/qec/lib/realtime/realtime_decoding.cpp @@ -13,24 +13,10 @@ #include "cudaq/qec/pcm_utils.h" #include "cudaq/qec/realtime/decoding_config.h" #include -#include -#include -#include -#include #include -#include #include #include -#ifdef CUDAQ_REALTIME_ROOT -#include "qec_realtime_session.h" -#include "rpc_producer.h" -#else -namespace cudaq::qec::realtime { -class qec_realtime_session {}; -} // namespace cudaq::qec::realtime -#endif - // Optional syndrome capture callback for --save_syndrome feature namespace { using SyndromeCaptureCallback = void (*)(const uint8_t *, size_t); @@ -38,103 +24,6 @@ SyndromeCaptureCallback g_syndrome_capture_callback = nullptr; } // namespace std::vector> g_decoders; -std::unique_ptr g_realtime_session; - -namespace { - -#ifdef CUDAQ_REALTIME_ROOT -inline cudaq_dispatch_launch_fn_t resolve_launch_dispatch_kernel_regular() { - return reinterpret_cast( - ::dlsym(RTLD_DEFAULT, "cudaq_launch_dispatch_kernel_regular")); -} -#endif - -bool realtime_mode_inproc_rpc_requested() { - const char *env = std::getenv("CUDAQ_QEC_REALTIME_MODE"); - if (!env || env[0] == '\0') - return false; - return std::strcmp(env, "inproc_rpc") == 0; -} - -bool any_decoder_supports_graph_dispatch() { - for (const auto &dec : g_decoders) { - if (dec && dec->supports_graph_dispatch()) - return true; - } - return false; -} - -} // namespace - -#ifdef CUDAQ_REALTIME_ROOT -namespace { - -void maybe_init_realtime_session() { - if (!realtime_mode_inproc_rpc_requested()) { - CUDA_QEC_INFO("CUDAQ_QEC_REALTIME_MODE not set to inproc_rpc; using " - "legacy direct-call decoding path."); - return; - } - - // Pick DEVICE vs HOST dispatch the same way qec_realtime_session does at - // initialize(): any graph-capable decoder => DEVICE mode (per-round - // GRAPH_LAUNCH enqueue + DEVICE_CALL get/reset, driven by the device dispatch - // kernel); otherwise HOST mode -- CPU decoders such as pymatching run all - // three RPCs inline on the CPU host loop. A mixed (graph + non-graph) set is - // rejected by qec_realtime_session::initialize() below. - const bool device_mode = any_decoder_supports_graph_dispatch(); - - cudaq_dispatch_launch_fn_t launch_fn = nullptr; - if (device_mode) { - // DEVICE mode needs the dispatch-kernel launch helper from - // libcudaq-realtime-dispatch.a (absorbed into the final executable). HOST - // mode uses no device launch helper. - launch_fn = resolve_launch_dispatch_kernel_regular(); - if (!launch_fn) - throw std::runtime_error( - "CUDAQ_QEC_REALTIME_MODE=inproc_rpc requested with a graph-capable " - "decoder but cudaq_launch_dispatch_kernel_regular could not be " - "resolved via dlsym(RTLD_DEFAULT, ...). The host executable must " - "absorb libcudaq-realtime-dispatch.a and link with " - "--export-dynamic."); - } else { - CUDA_QEC_INFO("CUDAQ_QEC_REALTIME_MODE=inproc_rpc with CPU (non-graph) " - "decoder(s); using HOST dispatch mode (no device kernel / no " - "device shared-ring setup)."); - } - - try { - g_realtime_session = - std::make_unique(g_decoders, - launch_fn); - g_realtime_session->initialize(); - } catch (const std::exception &e) { - const std::string what = e.what(); - g_realtime_session.reset(); - throw std::runtime_error("CUDAQ_QEC_REALTIME_MODE=inproc_rpc requested but " - "qec_realtime_session::initialize() threw: " + - what); - } -} - -void maybe_finalize_realtime_session() { - if (g_realtime_session) { - try { - g_realtime_session->finalize(); - } catch (const std::exception &e) { - CUDA_QEC_WARN("qec_realtime_session::finalize threw: {}", e.what()); - } - g_realtime_session.reset(); - } -} - -} // namespace -#else -namespace { -void maybe_init_realtime_session() {} -void maybe_finalize_realtime_session() {} -} // namespace -#endif // Helper to pack syndrome bits into bytes (8 bits per byte, MSB first for // readability) @@ -217,58 +106,6 @@ cudaqx::heterogeneous_map prepare_decoder_params( return params; } -std::unique_ptr create_realtime_decoder( - const cudaq::qec::decoding::config::decoder_config &decoder_config) { - if (decoder_config.id < 0 || static_cast(decoder_config.id) > - std::numeric_limits::max()) - throw std::invalid_argument("Decoder ID is outside the uint32_t range: " + - std::to_string(decoder_config.id)); - if (decoder_config.D_sparse.empty()) - throw std::runtime_error( - "D_sparse must be provided in decoder configuration"); - - auto t0 = std::chrono::high_resolution_clock::now(); - CUDA_QEC_INFO("Creating decoder {} of type {}", decoder_config.id, - decoder_config.type); - - auto pcm = cudaq::qec::pcm_from_sparse_vec(decoder_config.H_sparse, - decoder_config.syndrome_size, - decoder_config.block_size); - const auto num_observables = std::count(decoder_config.O_sparse.begin(), - decoder_config.O_sparse.end(), -1); - // Materialize O before decoder construction to validate its sparse shape and - // column indices for every decoder type. TRT also receives this matrix in its - // constructor parameters through prepare_decoder_params() below. - (void)cudaq::qec::pcm_from_sparse_vec( - decoder_config.O_sparse, num_observables, decoder_config.block_size); - auto decoder = cudaq::qec::get_decoder( - decoder_config.type, pcm, prepare_decoder_params(decoder_config)); - decoder->set_decoder_id(decoder_config.id); - decoder->set_O_sparse(decoder_config.O_sparse); - decoder->set_D_sparse(decoder_config.D_sparse); - - // Force plugin initialization before the caller publishes the decoder for - // realtime work. This preserves configure_decoders()'s existing behavior. - auto t1 = std::chrono::high_resolution_clock::now(); - std::vector syndrome(decoder_config.syndrome_size, 0.0); - decoder->decode(syndrome); - auto t2 = std::chrono::high_resolution_clock::now(); - std::chrono::duration creation_duration = t1 - t0; - std::chrono::duration initialization_duration = t2 - t1; - CUDA_QEC_INFO( - "Done initializing decoder {} in {:.6f} seconds (creation: {:.6f}s, " - "initial decoding dry run: {:.6f}s)", - decoder_config.id, - creation_duration.count() + initialization_duration.count(), - creation_duration.count(), initialization_duration.count()); - - return decoder; -} - -cudaq::qec::realtime::qec_realtime_session *get_realtime_session() { - return g_realtime_session.get(); -} - int configure_decoders( cudaq::qec::decoding::config::multi_decoder_config &config) { CUDA_QEC_INFO("Initializing decoders..."); @@ -301,77 +138,71 @@ int configure_decoders( return 3; } -#ifdef CUDAQ_REALTIME_ROOT - // inproc_rpc DEVICE sessions allocate pinned, device-mapped ring buffers - // (cudaHostAlloc(cudaHostAllocMapped) + cudaHostGetDevicePointer). - // cudaSetDeviceFlags(cudaDeviceMapHost) only takes effect BEFORE the device's - // CUDA context is created, and the per-decoder dry-run below - // (new_decoder->decode(...)) can create that context for GPU decoders -- so - // set the flag here, before any decoder is realized, rather than (only) later - // in qec_realtime_session::initialize(). Best-effort: if a context already - // exists this returns cudaErrorSetOnActiveProcess, which is harmless (mapped - // host allocation still works via UVA regardless of this device-wide flag), - // and HOST-mode CPU sessions do not use mapped memory at all. - if (realtime_mode_inproc_rpc_requested()) { - // The device-mapped ring buffers guarded by cudaDeviceMapHost are used only - // by the DEVICE-mode graph scheduler, which needs a usable GPU. CPU - // decoders run in HOST mode with plain host memory and never touch the - // device, so probe for a GPU first and skip the flag entirely when none is - // present. This keeps CPU-only / GPU-less machines from executing the - // device-flag call at all -- previously it ran unconditionally and logged a - // spurious "CUDA driver version is insufficient" warning. (If a graph - // decoder is later selected without a usable device, - // qec_realtime_session::initialize() still fails with a clear DEVICE-mode - // error.) - int device_count = 0; - cudaError_t count_err = cudaGetDeviceCount(&device_count); - if (count_err == cudaSuccess && device_count > 0) { - cudaError_t flags_err = cudaSetDeviceFlags(cudaDeviceMapHost); - if (flags_err != cudaSuccess && flags_err != cudaErrorSetOnActiveProcess) - CUDA_QEC_WARN( - "cudaSetDeviceFlags(cudaDeviceMapHost) returned '{}' before " - "decoder init; continuing (mapped alloc works via UVA).", - cudaGetErrorString(flags_err)); - } else { - // Reset the sticky runtime error so a later benign cudaGetLastError() - // isn't surprised by the no-device / insufficient-driver probe result. - cudaGetLastError(); - } - } -#endif - // Create the decoders based on the decoder configs. try { g_decoders.clear(); g_decoders.resize(max_decoder_id + 1); for (const auto &decoder_config : decoder_configs) { - g_decoders[decoder_config.id] = create_realtime_decoder(decoder_config); + // Form the PCM from the sparse vector. + auto t0 = std::chrono::high_resolution_clock::now(); + CUDA_QEC_INFO("Creating decoder {} of type {}", decoder_config.id, + decoder_config.type); + auto pcm = cudaq::qec::pcm_from_sparse_vec(decoder_config.H_sparse, + decoder_config.syndrome_size, + decoder_config.block_size); + auto new_decoder = cudaq::qec::get_decoder( + decoder_config.type, pcm, prepare_decoder_params(decoder_config)); + new_decoder->set_decoder_id(decoder_config.id); + // Count the number of -1's in the O_sparse vector. That is the number of + // rows (observables) in the observable matrix. + auto num_observables = std::count(decoder_config.O_sparse.begin(), + decoder_config.O_sparse.end(), -1); + // Populate the ***real-time*** fields of the decoder. + auto observable_matrix = cudaq::qec::pcm_from_sparse_vec( + decoder_config.O_sparse, num_observables, decoder_config.block_size); + new_decoder->set_O_sparse(decoder_config.O_sparse); + if (!decoder_config.D_sparse.empty()) { + new_decoder->set_D_sparse(decoder_config.D_sparse); + } else { + throw std::runtime_error( + "D_sparse must be provided in decoder configuration"); + } + + // Invoke a dummy decoding operation to force the decoder to be + // initialized. + auto t1 = std::chrono::high_resolution_clock::now(); + std::vector syndrome(decoder_config.syndrome_size, + 0.0); + new_decoder->decode(syndrome); + auto t2 = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration1 = t1 - t0; + std::chrono::duration duration2 = t2 - t1; + CUDA_QEC_INFO( + "Done initializing decoder {} in {:.6f} seconds (creation: {:.6f}s, " + "initial decoding dry run: {:.6f}s)", + decoder_config.id, duration1.count() + duration2.count(), + duration1.count(), duration2.count()); + + g_decoders[decoder_config.id] = std::move(new_decoder); } } catch (const std::exception &e) { CUDA_QEC_WARN("Error initializing decoders: {}", e.what()); return 4; } - maybe_init_realtime_session(); return 0; } void finalize_decoders() { CUDA_QEC_INFO("Finalizing the realtime decoding library."); - maybe_finalize_realtime_session(); g_decoders.clear(); } __attribute__((visibility("default"))) void -_set_syndrome_capture_callback(void (*callback)(const uint8_t *, size_t)) { +set_syndrome_capture_callback(void (*callback)(const uint8_t *, size_t)) { g_syndrome_capture_callback = callback; } -__attribute__((visibility("default"))) void (*_get_syndrome_capture_callback())( - const uint8_t *, size_t) { - return g_syndrome_capture_callback; -} - void enqueue_syndromes(std::size_t decoder_id, uint8_t *syndromes, std::uint64_t syndrome_length, std::uint64_t tag) { if (decoder_id >= g_decoders.size()) { @@ -410,22 +241,6 @@ void enqueue_syndromes(std::size_t decoder_id, uint8_t *syndromes, } }; -#ifdef CUDAQ_REALTIME_ROOT - if (g_realtime_session) { - capture_syndromes(); - try { - cudaq::qec::decoding::rpc_producer::enqueue_syndromes( - *g_realtime_session, decoder_id, syndromes, syndrome_length, tag); - } catch ( - const cudaq::qec::decoding::rpc_producer::dispatcher_unresponsive_error - &) { - maybe_finalize_realtime_session(); - throw; - } - return; - } -#endif - // Direct-call path: this caller thread runs the decode, but // configure_decoders() constructed every decoder sequentially on one thread, // leaving the LAST decoder's device current. Point the thread at this @@ -484,22 +299,6 @@ void get_corrections(std::size_t decoder_id, uint8_t *corrections, correction_length, num_observables)); } -#ifdef CUDAQ_REALTIME_ROOT - if (g_realtime_session) { - try { - cudaq::qec::decoding::rpc_producer::get_corrections( - *g_realtime_session, decoder_id, corrections, correction_length, - reset ? 1u : 0u); - } catch ( - const cudaq::qec::decoding::rpc_producer::dispatcher_unresponsive_error - &) { - maybe_finalize_realtime_session(); - throw; - } - return; - } -#endif - // clear_corrections may touch device memory in some plugins. cudaq::qec::detail_affinity::set_cuda_device_for_decode( decoder->get_cuda_device_id()); @@ -523,21 +322,6 @@ void reset_decoder(std::size_t decoder_id) { fmt::format("Decoder {} not found", decoder_id)); } -#ifdef CUDAQ_REALTIME_ROOT - if (g_realtime_session) { - try { - cudaq::qec::decoding::rpc_producer::reset_decoder(*g_realtime_session, - decoder_id); - } catch ( - const cudaq::qec::decoding::rpc_producer::dispatcher_unresponsive_error - &) { - maybe_finalize_realtime_session(); - throw; - } - return; - } -#endif - cudaq::qec::detail_affinity::set_cuda_device_for_decode( decoder->get_cuda_device_id()); decoder->reset_decoder(); diff --git a/libs/qec/lib/realtime/realtime_decoding.h b/libs/qec/lib/realtime/realtime_decoding.h index d4e6558f7..c41669e22 100644 --- a/libs/qec/lib/realtime/realtime_decoding.h +++ b/libs/qec/lib/realtime/realtime_decoding.h @@ -8,26 +8,13 @@ #pragma once -#include "cudaq/qec/decoder.h" #include "cudaq/qec/realtime/decoding_config.h" #include -#include // Note: none of these are intended to be user-facing functions. -namespace cudaq::qec::realtime { -class qec_realtime_session; -} // namespace cudaq::qec::realtime - namespace cudaq::qec::decoding::host { -/// @brief Accessor for the per-process realtime session. Returns nullptr -/// unless CUDAQ_QEC_REALTIME_MODE=inproc_rpc has initialized the shared-ring -/// dispatch session. -__attribute__((visibility("default"))) -cudaq::qec::realtime::qec_realtime_session * -get_realtime_session(); - __attribute__((visibility("default"))) void enqueue_syndromes(std::size_t decoder_id, uint8_t *syndromes, std::uint64_t syndrome_length, std::uint64_t tag); @@ -36,17 +23,6 @@ __attribute__((visibility("default"))) cudaqx::heterogeneous_map prepare_decoder_params( const cudaq::qec::decoding::config::decoder_config &decoder_config); -/// Construct and initialize one decoder for realtime use. The returned decoder -/// is fully configured with its ID and O/D matrices, but is not installed in a -/// process-global registry or attached to a worker thread. -/// -/// @throws std::invalid_argument if the decoder ID cannot be represented. -/// @throws std::runtime_error if required realtime configuration is missing or -/// decoder construction/initialization fails. -__attribute__((visibility("default"))) std::unique_ptr -create_realtime_decoder( - const cudaq::qec::decoding::config::decoder_config &decoder_config); - __attribute__((visibility("default"))) void get_corrections(std::size_t decoder_id, uint8_t *corrections, std::uint64_t correction_length, bool reset); @@ -65,12 +41,6 @@ void finalize_decoders(); /// @param callback Function pointer that receives packed syndrome bytes. /// Set to nullptr to disable capture. __attribute__((visibility("default"))) void -_set_syndrome_capture_callback(void (*callback)(const uint8_t *, size_t)); - -/// @brief The currently registered syndrome-capture callback (nullptr if -/// none). Served decode paths that bypass host::enqueue_syndromes (the -/// decoding-server service) use this to keep --save_syndrome working. -__attribute__((visibility("default"))) void (*_get_syndrome_capture_callback())( - const uint8_t *, size_t); +set_syndrome_capture_callback(void (*callback)(const uint8_t *, size_t)); } // namespace cudaq::qec::decoding::host diff --git a/libs/qec/lib/realtime/rpc_producer.cpp b/libs/qec/lib/realtime/rpc_producer.cpp index ecbbfff85..05ca34a73 100644 --- a/libs/qec/lib/realtime/rpc_producer.cpp +++ b/libs/qec/lib/realtime/rpc_producer.cpp @@ -12,117 +12,77 @@ #include "qec_realtime_session.h" #include "cudaq/qec/realtime/decoder_rpc_ids.h" -#include "cudaq/realtime/daemon/dispatcher/cudaq_realtime.h" -#include "cudaq/realtime/daemon/dispatcher/dispatch_kernel_launch.h" #include -#include #include #include -#include -#include -#include // for usleep +#include #include namespace cudaq::qec::decoding::rpc_producer { - namespace { -// Process-wide monotonic counter for `RPCHeader::request_id`. The wire -// protocol echoes request_id in `RPCResponse::request_id`; today we don't -// match-by-id (we wait on the magic of the slot we wrote to), but the -// handlers still validate it's nonzero, and DEVICE_LOOP stats keys off it. std::atomic g_request_id_counter{1}; +std::atomic g_next_slot_hint{0}; + +// Producer-owned "busy" sentinel written into tx_flags while a slot is held. +// The host loop runs with skip_tx_markers=1 and never reads/writes tx_flags, so +// the producer repurposes it as an ownership token (any non-zero value works). +constexpr std::uint64_t kSlotBusyMarker = ~std::uint64_t{0}; std::uint32_t next_request_id() { return g_request_id_counter.fetch_add(1, std::memory_order_relaxed); } -// Enforces the single-producer contract documented in rpc_producer.h. The -// producer path is single-producer by design -- the sole caller is the single- -// threaded QEC decode loop. acquire_slot() selects a free slot without an -// atomic reservation, so two concurrent producers could pick the same slot and -// corrupt each other's RPC. Rather than pay for full multi-producer support -// (CAS slot-claim / per-producer arena), we DETECT a contract violation and -// fail loudly. This is a real throw, NOT assert(): release builds compile -// assert() out, so an assert would enforce nothing in production. -std::atomic g_producer_active{false}; - -struct single_producer_guard { - single_producer_guard() { - bool expected = false; - if (!g_producer_active.compare_exchange_strong(expected, true, - std::memory_order_acquire)) - throw std::runtime_error( - "rpc_producer: concurrent producer detected. This RPC path is " - "single-producer (the single-threaded QEC decode loop); serialize " - "calls or add multi-producer support (CAS slot-claim / per-producer " - "arena)."); - } - ~single_producer_guard() { - g_producer_active.store(false, std::memory_order_release); - } - single_producer_guard(const single_producer_guard &) = delete; - single_producer_guard &operator=(const single_producer_guard &) = delete; -}; - -// Bounded spin for the NEXT slot in monotonic ring order. Returns -// UINT32_MAX on timeout. -// -// Ring discipline: the producer walks slots monotonically -// (session.producer_cursor(), advanced mod num_slots each acquire) rather -// than picking the lowest free slot. This keeps it in lockstep with the -// strict-FIFO consumer (the device-graph scheduler / host loop, both with -// shared-ring scanning OFF), which waits at exactly its own monotonically- -// advancing cursor. The cursor is reset to 0 by -// qec_realtime_session::initialize(), matching the consumer's reset, so both -// start at slot 0. Back-pressure is preserved: we wait until the chosen slot -// is free (rx_flags[s]==0 AND tx_flags[s]==0 -- request consumed by the -// dispatcher and response consumed by release_slot()). -// -// THREAD-SAFETY ASSUMPTION (single producer): -// We read-modify-write producer_cursor() non-atomically, which is correct -// under the single-producer invariant documented in rpc_producer.h (the QEC -// main loop is the only producer, single-threaded by construction). A future -// multi-producer design would make the cursor advance an atomic fetch-add -// (the natural multi-producer ring head) and add a CAS claim of the slot; -// this is deferred to a follow-up MR. std::uint32_t acquire_slot(cudaq::qec::realtime::qec_realtime_session &session, int timeout_ms) { volatile std::uint64_t *rx = session.rx_flags_host(); volatile std::uint64_t *tx = session.tx_flags_host(); - const std::size_t n = session.num_slots(); - if (rx == nullptr || tx == nullptr || n == 0) - return UINT32_MAX; - const std::uint32_t s = - static_cast(session.producer_cursor() % n); for (int waited = 0; waited < timeout_ms; ++waited) { - if (rx[s] == 0 && tx[s] == 0) { - session.set_producer_cursor((s + 1u) % n); - return s; + const std::uint32_t start = + g_next_slot_hint.load(std::memory_order_relaxed) % + static_cast(session.num_slots()); + for (std::uint32_t s = 0; s < session.num_slots(); ++s) { + const std::uint32_t slot = + static_cast((start + s) % session.num_slots()); + // A slot is free only when no request is in flight (rx == 0). Claim it + // by atomically flipping tx_flags 0 -> BUSY: this reserves the TX + // response buffer for the entire acquire..release_slot window rather than + // only until the host loop clears rx_flags, and the CAS makes the claim + // safe against concurrent producers. tx == 0 only happens after + // release_slot (which runs after the host loop consumed the request and + // cleared rx), so rx is already 0 at a successful claim. + if (rx[slot] != 0) + continue; + std::uint64_t expected = 0; + if (__atomic_compare_exchange_n(&tx[slot], &expected, kSlotBusyMarker, + false, __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED)) { + g_next_slot_hint.store( + static_cast((slot + 1) % session.num_slots()), + std::memory_order_relaxed); + return slot; + } } - // 1 ms granularity matches the test's spin cadence. The shared ring - // is host-pinned + UVA-mapped, so the producer's view of rx/tx flags - // is coherent with the GPU consumer's writes after a __sync_- - // synchronize on the consumer side; usleep here keeps the busy-wait - // off the critical path. usleep(1000); } return UINT32_MAX; } -// Write an RPC request into `slot` and publish it by writing the device- -// visible slot address into rx_flags[slot]. Mirrors WriteAndSignal in the -// test; the slot is pre-acquired so the caller owns it from acquire_slot() -// through release_slot(). +void require_initialized(cudaq::qec::realtime::qec_realtime_session &session, + const char *fn) { + if (!session.initialized()) { + std::ostringstream os; + os << "rpc_producer::" << fn + << ": session is not initialized; call configure_decoders first"; + throw std::runtime_error(os.str()); + } +} + void write_and_signal(cudaq::qec::realtime::qec_realtime_session &session, std::uint32_t slot, std::uint32_t function_id, std::uint32_t request_id, const void *payload, std::size_t payload_len) { - // Two-ring wire format: requests go into the RX backing, responses - // come back via the TX backing. We zero just the RX slot here; the - // TX slot is cleared lazily on release_slot() so a stale response - // body from a previous round doesn't confound the next reader. std::uint8_t *rx_slot_host = session.rx_data_host() + slot * session.slot_size(); std::memset(rx_slot_host, 0, session.slot_size()); @@ -135,312 +95,192 @@ void write_and_signal(cudaq::qec::realtime::qec_realtime_session &session, std::memcpy(rx_slot_host + sizeof(cudaq::realtime::RPCHeader), payload, payload_len); __sync_synchronize(); - // Address-as-flag publish: the DEVICE-visible RX slot address is what - // the dispatcher polls for on rx_flags. Same UVA on host and device - // because the RX data backing is pinned+mapped. - session.rx_flags_host()[slot] = reinterpret_cast( - session.rx_data_dev() + slot * session.slot_size()); + session.rx_flags_host()[slot] = reinterpret_cast(rx_slot_host); } -// Bounded spin for response publication. Returns false on timeout. The -// response is considered complete only after the writer has produced an -// RPCResponse header and published the matching tx_flags entry. bool wait_for_response(cudaq::qec::realtime::qec_realtime_session &session, std::uint32_t slot, int timeout_ms) { - // Two-ring wire format: the response lives in the TX slot. The - // dispatcher's writer (the captured graph for GRAPH_LAUNCH; the - // DEVICE_LOOP kernel for DEVICE_CALL) writes RPCResponse and then signals - // tx_flags[slot]. Wait for both before reading or releasing the slot. std::uint8_t *tx_slot_host = session.tx_data_host() + slot * session.slot_size(); auto *resp = reinterpret_cast(tx_slot_host); for (int waited = 0; waited < timeout_ms; ++waited) { - __sync_synchronize(); - if (resp->magic == cudaq::realtime::RPC_MAGIC_RESPONSE && - session.tx_flags_host()[slot] != 0) + std::uint32_t magic = 0; + __atomic_load(&resp->magic, &magic, __ATOMIC_ACQUIRE); + if (magic == cudaq::realtime::RPC_MAGIC_RESPONSE) return true; - // 200us granularity matches the test. Shorter than acquire_slot's - // sleep because get_corrections / reset round-trips are sub-ms on - // typical GPUs and a 1ms cadence would dominate the round-trip - // budget for small per-shot payloads. usleep(200); } return false; } -// Release a slot after the caller has finished consuming the response. -// Two-ring wire format: clears the TX backing (so the next reader of -// this slot won't see a stale `RPC_MAGIC_RESPONSE`). The RX backing -// for this slot was already overwritten by the dispatcher (the captured -// graph) when it parsed the request; we zero it again defensively in -// write_and_signal() before the next request, so we don't need to wipe -// it here. Clears tx_flags[slot] to unblock acquire_slot for the next -// caller. rx_flags is cleared by the dispatcher (see -// host_dispatcher.cu::finish_slot_and_advance), not by the producer. void release_slot(cudaq::qec::realtime::qec_realtime_session &session, std::uint32_t slot) { - std::uint8_t *tx_slot_host = - session.tx_data_host() + slot * session.slot_size(); - std::memset(tx_slot_host, 0, session.slot_size()); + // rx_flags[slot] is cleared by cudaq_host_dispatcher_loop after it consumes + // the request. acquire_slot set tx_flags[slot] to the BUSY token to reserve + // the response buffer; clear it here (after reading the response) to return + // the slot to the pool. Order the TX data wipe before the token release so a + // re-acquiring producer never observes stale response bytes. + std::memset(session.tx_data_host() + slot * session.slot_size(), 0, + session.slot_size()); __sync_synchronize(); session.tx_flags_host()[slot] = 0; } -// Common pre-flight: session must be initialized + ring must be live. -// Pulled into a helper so each public function's pre-flight error is -// uniform (and so the runtime error includes the function name). -void require_initialized(cudaq::qec::realtime::qec_realtime_session &session, - const char *fn) { - if (!session.initialized()) { +const cudaq::realtime::RPCResponse * +checked_response(cudaq::qec::realtime::qec_realtime_session &session, + std::uint32_t slot, std::uint32_t request_id, const char *fn) { + auto *resp = reinterpret_cast( + session.tx_data_host() + slot * session.slot_size()); + if (resp->request_id != request_id) { std::ostringstream os; - os << "cudaq::qec::decoding::rpc_producer::" << fn - << ": session is not initialized(). Call qec_realtime_session::" - "initialize() before sending RPCs."; + os << "rpc_producer::" << fn << ": response request_id mismatch (got " + << resp->request_id << ", expected " << request_id << ")"; throw std::runtime_error(os.str()); } - if (session.rx_flags_host() == nullptr || - session.tx_flags_host() == nullptr || session.rx_data_host() == nullptr || - session.rx_data_dev() == nullptr || session.tx_data_host() == nullptr || - session.tx_data_dev() == nullptr) { + if (resp->status != 0) { std::ostringstream os; - os << "cudaq::qec::decoding::rpc_producer::" << fn - << ": session ring buffer pointers are null even though " - "initialized()==true. This indicates a teardown race or a " - "double-finalize bug."; + os << "rpc_producer::" << fn << ": RPC status " << resp->status; throw std::runtime_error(os.str()); } + return resp; } } // namespace void enqueue_syndromes(cudaq::qec::realtime::qec_realtime_session &session, std::size_t decoder_id, const std::uint8_t *syndromes, - std::uint64_t num_syndromes, std::uint64_t tag) { - single_producer_guard producer_guard; + std::uint64_t num_syndromes, std::uint64_t counter, + std::uint64_t syndrome_mapping_id) { + namespace rpc = cudaq::qec::decoding::rpc; require_initialized(session, "enqueue_syndromes"); + if (syndromes == nullptr && num_syndromes != 0) + throw std::runtime_error("rpc_producer::enqueue_syndromes: null syndromes"); - if (syndromes == nullptr && num_syndromes > 0) - throw std::runtime_error( - "rpc_producer::enqueue_syndromes: syndromes == nullptr but " - "num_syndromes > 0"); - - // Build the wire payload per decoder_server_runtime.md#enqueue_syndromes: - // 32-byte EnqueueRequestPayload (decoder_id, counter, - // syndrome_mapping_id, num_syndromes; all INT64) - // + ceil(num_syndromes/8) bit-packed syndrome bytes (LSB-first), no pad. - const std::size_t bp_bytes = - cudaq::qec::decoding::rpc::bit_packed_bytes(num_syndromes); + const std::size_t bit_bytes = rpc::bit_packed_bytes(num_syndromes); const std::size_t body_bytes = - sizeof(cudaq::qec::decoding::rpc::EnqueueRequestPayload) + bp_bytes; + rpc::align_to_8(sizeof(rpc::EnqueueRequestPayload) + bit_bytes); std::vector payload(body_bytes, 0); - auto *p = - reinterpret_cast( - payload.data()); - p->decoder_id = static_cast(decoder_id); - p->counter = static_cast(tag); - p->syndrome_mapping_id = 0; - p->num_syndromes = static_cast(num_syndromes); - if (num_syndromes > 0) { - std::uint8_t *bits = - payload.data() + - sizeof(cudaq::qec::decoding::rpc::EnqueueRequestPayload); - for (std::uint64_t i = 0; i < num_syndromes; ++i) { - // Source format: one bit per byte (low bit significant), matching - // the existing plugin / test contract. Bit i lands at bit (i mod - // 8) of bits[i / 8] -- LSB-first, per the spec. - if (syndromes[i] & 0x1u) - bits[i / 8] |= - static_cast(1u << static_cast(i % 8)); - } + auto *body = reinterpret_cast(payload.data()); + body->decoder_id = static_cast(decoder_id); + body->counter = static_cast(counter); + body->syndrome_mapping_id = static_cast(syndrome_mapping_id); + body->num_syndromes = static_cast(num_syndromes); + auto *bits = payload.data() + sizeof(rpc::EnqueueRequestPayload); + for (std::uint64_t i = 0; i < num_syndromes; ++i) { + if (syndromes[i] & 0x1u) + bits[i >> 3] |= static_cast(1u << (i & 7)); } - std::uint32_t slot = acquire_slot(session, kAcquireSlotTimeoutMs); + const std::uint32_t request_id = next_request_id(); + const std::uint32_t slot = acquire_slot(session, kAcquireSlotTimeoutMs); if (slot == UINT32_MAX) throw dispatcher_unresponsive_error( "rpc_producer::enqueue_syndromes: timed out acquiring a free slot"); - // request_id is for correlation. The full-width application tag travels as - // payload arg1 (`counter`); request_id is a 32-bit window so use the low 32 - // bits of tag. When `tag = (shot << 16) | round` (the production - // convention), this yields a unique-per-shot request_id within a 65k-shot - // window, more than enough for in-flight correlation in DEVICE_LOOP stats / - // debugging. - const std::uint32_t request_id = static_cast(tag); - write_and_signal(session, slot, - cudaq::qec::decoding::rpc::kEnqueueSyndromesFunctionId, - request_id, payload.data(), payload.size()); + write_and_signal(session, slot, rpc::kEnqueueSyndromesFunctionId, request_id, + payload.data(), payload.size()); if (!wait_for_response(session, slot, kResponseTimeoutMs)) { release_slot(session, slot); - std::ostringstream os; - os << "rpc_producer::enqueue_syndromes: timed out waiting for ACK " - "(decoder_id=" - << decoder_id << ", tag=0x" << std::hex << tag << ")"; - throw dispatcher_unresponsive_error(os.str()); + throw dispatcher_unresponsive_error( + "rpc_producer::enqueue_syndromes: timed out waiting for response"); } - - std::uint8_t *tx_slot_host = - session.tx_data_host() + slot * session.slot_size(); - const auto *resp = - reinterpret_cast(tx_slot_host); - - if (resp->status != 0) { - const std::int32_t status = resp->status; + const cudaq::realtime::RPCResponse *resp = nullptr; + try { + resp = checked_response(session, slot, request_id, "enqueue_syndromes"); + } catch (...) { release_slot(session, slot); - std::ostringstream os; - os << "rpc_producer::enqueue_syndromes: non-zero status (" << status - << ") for decoder_id=" << decoder_id << ", tag=0x" << std::hex << tag; - throw std::runtime_error(os.str()); + throw; } - // Per decoder_server_runtime.md the dispatcher always emits a 24-byte - // RPCResponse, even for fire-and-forget calls; the body is empty - // (result_len == 0). Drop the ACK and release the slot. if (resp->result_len != 0) { - const std::uint32_t got = resp->result_len; release_slot(session, slot); - std::ostringstream os; - os << "rpc_producer::enqueue_syndromes: unexpected non-empty ACK " - "(decoder_id=" - << decoder_id << "), expected result_len=0, got " << got; - throw std::runtime_error(os.str()); + throw std::runtime_error( + "rpc_producer::enqueue_syndromes: expected empty ACK response"); } - release_slot(session, slot); } void get_corrections(cudaq::qec::realtime::qec_realtime_session &session, std::size_t decoder_id, std::uint8_t *corrections, std::uint64_t correction_length, std::uint64_t reset) { - single_producer_guard producer_guard; + namespace rpc = cudaq::qec::decoding::rpc; require_initialized(session, "get_corrections"); + if (!corrections && correction_length != 0) + throw std::runtime_error("rpc_producer::get_corrections: null corrections"); - if (corrections == nullptr && correction_length > 0) - throw std::runtime_error( - "rpc_producer::get_corrections: corrections == nullptr but " - "correction_length > 0"); - - // Build the wire payload per decoder_server_runtime.md#get_corrections: - // 17 bytes total: decoder_id (INT64) + return_size (INT64, the OUT - // std::vector length) + reset (UINT8, trailing bool, no pad). The - // struct is laid out exactly this way. - cudaq::qec::decoding::rpc::GetCorrectionsRequestPayload payload{}; + rpc::GetCorrectionsRequestPayload payload{}; payload.decoder_id = static_cast(decoder_id); payload.return_size = static_cast(correction_length); - payload.reset = reset ? std::uint8_t{1} : std::uint8_t{0}; + payload.reset = reset ? 1 : 0; - std::uint32_t slot = acquire_slot(session, kAcquireSlotTimeoutMs); + const std::uint32_t request_id = next_request_id(); + const std::uint32_t slot = acquire_slot(session, kAcquireSlotTimeoutMs); if (slot == UINT32_MAX) throw dispatcher_unresponsive_error( "rpc_producer::get_corrections: timed out acquiring a free slot"); - const std::uint32_t request_id = next_request_id(); - write_and_signal(session, slot, - cudaq::qec::decoding::rpc::kGetCorrectionsFunctionId, - request_id, &payload, sizeof(payload)); - + write_and_signal(session, slot, rpc::kGetCorrectionsFunctionId, request_id, + &payload, sizeof(payload)); if (!wait_for_response(session, slot, kResponseTimeoutMs)) { release_slot(session, slot); - std::ostringstream os; - os << "rpc_producer::get_corrections: timed out waiting for response " - "(decoder_id=" - << decoder_id << ")"; - throw dispatcher_unresponsive_error(os.str()); + throw dispatcher_unresponsive_error( + "rpc_producer::get_corrections: timed out waiting for response"); } - - std::uint8_t *tx_slot_host = - session.tx_data_host() + slot * session.slot_size(); - const auto *resp = - reinterpret_cast(tx_slot_host); - - if (resp->status != 0) { - const std::int32_t status = resp->status; + const cudaq::realtime::RPCResponse *resp = nullptr; + try { + resp = checked_response(session, slot, request_id, "get_corrections"); + } catch (...) { release_slot(session, slot); - std::ostringstream os; - os << "rpc_producer::get_corrections: non-zero status (" << status - << ") for decoder_id=" << decoder_id; - throw std::runtime_error(os.str()); + throw; } - // Per spec result_len = ceil(R/8) exactly (no trailing pad). - const std::size_t expected_bp = - cudaq::qec::decoding::rpc::bit_packed_bytes(correction_length); - if (resp->result_len != static_cast(expected_bp)) { - const std::uint32_t got = resp->result_len; + const std::size_t expected_len = + rpc::align_to_8(rpc::bit_packed_bytes(correction_length)); + if (resp->result_len != expected_len) { release_slot(session, slot); - std::ostringstream os; - os << "rpc_producer::get_corrections: result_len mismatch (decoder_id=" - << decoder_id << "), expected " << expected_bp - << " (ceil(R/8) for R=" << correction_length << "), got " << got; - throw std::runtime_error(os.str()); - } - - if (correction_length > 0) { - // Unpack the bit-packed result (LSB-first) into the caller's - // byte-per-bit output buffer to preserve the API surface used by - // realtime_decoding.cpp / the ABI seen by test code. Bit i of the - // correction vector lives at bit (i mod 8) of bits[i/8]. - const std::uint8_t *bits = - tx_slot_host + sizeof(cudaq::realtime::RPCResponse); - for (std::uint64_t i = 0; i < correction_length; ++i) { - corrections[i] = static_cast( - (bits[i / 8] >> static_cast(i % 8)) & 0x1u); - } + throw std::runtime_error( + "rpc_producer::get_corrections: malformed result_len"); } + const std::uint8_t *bits = session.tx_data_host() + + slot * session.slot_size() + + sizeof(cudaq::realtime::RPCResponse); + for (std::uint64_t i = 0; i < correction_length; ++i) + corrections[i] = (bits[i >> 3] >> (i & 7)) & 0x1u; release_slot(session, slot); } void reset_decoder(cudaq::qec::realtime::qec_realtime_session &session, std::size_t decoder_id) { - single_producer_guard producer_guard; + namespace rpc = cudaq::qec::decoding::rpc; require_initialized(session, "reset_decoder"); - - cudaq::qec::decoding::rpc::ResetRequestPayload payload{}; + rpc::ResetRequestPayload payload{}; payload.decoder_id = static_cast(decoder_id); - std::uint32_t slot = acquire_slot(session, kAcquireSlotTimeoutMs); + const std::uint32_t request_id = next_request_id(); + const std::uint32_t slot = acquire_slot(session, kAcquireSlotTimeoutMs); if (slot == UINT32_MAX) throw dispatcher_unresponsive_error( "rpc_producer::reset_decoder: timed out acquiring a free slot"); - const std::uint32_t request_id = next_request_id(); - write_and_signal(session, slot, - cudaq::qec::decoding::rpc::kResetDecoderFunctionId, - request_id, &payload, sizeof(payload)); - + write_and_signal(session, slot, rpc::kResetDecoderFunctionId, request_id, + &payload, sizeof(payload)); if (!wait_for_response(session, slot, kResponseTimeoutMs)) { release_slot(session, slot); - std::ostringstream os; - os << "rpc_producer::reset_decoder: timed out waiting for response " - "(decoder_id=" - << decoder_id << ")"; - throw dispatcher_unresponsive_error(os.str()); + throw dispatcher_unresponsive_error( + "rpc_producer::reset_decoder: timed out waiting for response"); } - - std::uint8_t *tx_slot_host = - session.tx_data_host() + slot * session.slot_size(); - const auto *resp = - reinterpret_cast(tx_slot_host); - - if (resp->status != 0) { - const std::int32_t status = resp->status; + const cudaq::realtime::RPCResponse *resp = nullptr; + try { + resp = checked_response(session, slot, request_id, "reset_decoder"); + } catch (...) { release_slot(session, slot); - std::ostringstream os; - os << "rpc_producer::reset_decoder: non-zero status (" << status - << ") for decoder_id=" << decoder_id; - throw std::runtime_error(os.str()); + throw; } - // Per spec the dispatcher always emits an empty 24-byte RPCResponse for - // fire-and-forget reset; result_len must be 0. if (resp->result_len != 0) { - const std::uint32_t got = resp->result_len; release_slot(session, slot); - std::ostringstream os; - os << "rpc_producer::reset_decoder: unexpected non-empty ACK " - "(decoder_id=" - << decoder_id << "), expected result_len=0, got " << got; - throw std::runtime_error(os.str()); + throw std::runtime_error( + "rpc_producer::reset_decoder: expected empty ACK response"); } - release_slot(session, slot); } diff --git a/libs/qec/lib/realtime/rpc_producer.h b/libs/qec/lib/realtime/rpc_producer.h index f390431d0..a1126a001 100644 --- a/libs/qec/lib/realtime/rpc_producer.h +++ b/libs/qec/lib/realtime/rpc_producer.h @@ -18,167 +18,26 @@ class qec_realtime_session; namespace cudaq::qec::decoding::rpc_producer { -/// @brief Thrown when an RPC round-trip cannot make progress because the -/// dispatcher is unresponsive -- either no free slot appears within -/// `kAcquireSlotTimeoutMs`, or no `RPCResponse` appears within -/// `kResponseTimeoutMs`. Distinct from `std::runtime_error` so the host-side -/// caller (realtime_decoding.cpp) can recognize it as fatal-to-the-session -/// (the device kernel / host monitor has stopped servicing the ring) and tear -/// the session down rather than retry into a slow ghost-slot leak. All other -/// producer errors (bad args, non-zero RPC status, malformed result_len) -/// stay plain `std::runtime_error`. struct dispatcher_unresponsive_error : std::runtime_error { using std::runtime_error::runtime_error; }; -//============================================================================== -// Host-side RPC producer for the inproc_rpc realtime decoding path. -// -// These three functions are the host-process counterparts to the three RPC -// handlers wired up by qec_realtime_session + decoder_rpc_dispatch.cu, all -// conformant with proposals/decoder_server_runtime.md: -// - enqueue_syndromes GRAPH_LAUNCH; function_id == -// fnv1a("enqueue_syndromes") -// == kEnqueueSyndromesFunctionId; one captured graph -// per decoder; host monitor sub-routes by -// (function_id, routing_key=decoder_id). -// - get_corrections DEVICE_CALL; function_id == fnv1a("get_corrections") -// == kGetCorrectionsFunctionId. -// - reset_decoder DEVICE_CALL; function_id == fnv1a("reset_decoder") -// == kResetDecoderFunctionId. -// -// All three RPCs are request/response on the wire: the dispatcher always -// emits a 24-byte RPCResponse (status=0, result_len=0 for the two fire- -// and-forget calls; bit-packed correction bytes + 0..7 pad for get_- -// corrections). Producers wait for the ACK and drop the response body -// for enqueue/reset. -// -// Each function: -// 1. Resolves the per-process session via the `session` argument (no global -// lookup) so test code can drive a private session without touching -// `g_realtime_session`. -// 2. Acquires a free RX slot from the ring buffer. -// 3. Writes RPCHeader + the matching wire-format payload from -// decoder_rpc_ids.h. Payload scalars are INT64; bools are UINT8+pad; -// bit-packed arrays are LSB-first; whole payload is padded to an -// 8-byte multiple. -// 4. Publishes the slot by writing the DEVICE-visible RX slot address -// into rx_flags[slot] (the "address-as-flag" convention of the shared -// ring; both the HOST_LOOP and DEVICE_LOOP dispatchers expect this). -// 5. Spins on RPCResponse::magic, checks status, and (for -// get_corrections) reads back the bit-packed result. -// 6. Releases the slot (clears rx_flags + tx_flags + the slot's first -// bytes so the response magic doesn't get misread on slot reuse). -// -// On any error (no slot within timeout_ms, response timeout, non-zero -// RPC status, malformed result_len), the corresponding function throws -// `std::runtime_error` with a contextual message. The host-side caller in -// the production path (realtime_decoding.cpp::enqueue_syndromes / -// get_corrections / reset) is also a free function in -// `cudaq::qec::decoding::host`, so it gets to choose how to surface the -// error -- e.g. by logging then rethrowing. -// -// THREAD-SAFETY: -// All three functions are SINGLE-PRODUCER -- this is a hard contract, not -// merely an assumption. The production caller in realtime_decoding.cpp:: -// enqueue_syndromes / get_corrections / reset_decoder is invoked from the QEC -// main loop, which is single-threaded by construction. The contract is now -// ENFORCED at runtime: each function holds a single_producer_guard that throws -// if a second producer is active concurrently (an always-on check -- a real -// throw, not assert(), so it stays active in release builds). Full multi- -// producer support (CAS slot-claim or a per-producer arena) remains a -// deliberate follow-up. -// -// acquire_slot() in rpc_producer.cpp scans for a free slot (rx_flags == -// tx_flags == 0) but does NOT atomically claim it on return. That's -// fine while there's a single producer (the chosen slot is written -// before the next call to acquire_slot()), but it would race if a -// second producer thread were ever introduced. Multi-producer support -// is a deliberate follow-up: it requires either (a) a CAS-based atomic -// claim on the rx_flags slot ("0 -> in-progress") inside acquire_slot, -// or (b) a per-producer arena of slots in the ring. See -// `acquire_slot()` for the inline assumption comment. -// -// As a separate constraint, the *handlers* themselves are not -// re-entrant per-decoder (the plugin's GpuDecoderState is shared -// across in-flight enqueue rounds for that decoder_id), so the -// wire-level contract is "one in-flight RPC per (decoder_id, -// function_id) on the wire at a time" regardless of producer count. -//============================================================================== +constexpr int kAcquireSlotTimeoutMs = 5000; +constexpr int kResponseTimeoutMs = 5000; -/// @brief Send a per-round enqueue RPC for `decoder_id` carrying -/// `num_syndromes` syndrome bits, bit-packed LSB-first into the wire payload. -/// -/// The session holds N GRAPH_LAUNCH entries that all share the canonical -/// `kEnqueueSyndromesFunctionId`; the host monitor disambiguates them by -/// `routing_key = decoder_id` (see proposals/cudaq_realtime_host_api.bs# -/// host-path-graph-routing-key). The dispatcher always emits a 24-byte -/// `RPCResponse` with `status=0, result_len=0` (no body); this producer -/// waits for the ACK and drops it. -/// -/// @param session Realtime session. Must be `initialized()`. -/// @param decoder_id Routing key. Written into payload arg0 AND -/// matched against the function table's -/// `routing_key` field. -/// @param syndromes Pointer to `syndrome_length` raw syndrome bytes, -/// one bit-per-byte at the source (each input byte -/// contributes one bit to the wire-side bit-packed -/// array). -/// @param num_syndromes Number of syndrome BITS for this round -/// (== `syndrome_length`). Written into payload -/// arg3. -/// @param tag Application-level breadcrumb written into -/// `RPCHeader::request_id` (low 32 bits) and payload -/// arg1 (`counter`, full 64 bits). This first pass -/// always emits payload arg2 (`syndrome_mapping_id`) as -/// 0 for contiguous identity mapping. Production callers -/// typically pack `(shot << 16) | round` into `tag`. __attribute__((visibility("default"))) void enqueue_syndromes(cudaq::qec::realtime::qec_realtime_session &session, std::size_t decoder_id, const std::uint8_t *syndromes, - std::uint64_t num_syndromes, std::uint64_t tag); + std::uint64_t num_syndromes, std::uint64_t counter, + std::uint64_t syndrome_mapping_id); -/// @brief Fetch `correction_length` correction bytes for `decoder_id` and -/// optionally reset the device-side accumulated correction buffer. -/// -/// Handler is the shared DEVICE_CALL `get_corrections_ui64`; routing happens -/// in `decoder_rpc_dispatch.cu` based on `decoder_id` in the payload. -/// -/// @param session Realtime session. Must be `initialized()`. -/// @param decoder_id Decoder index for state lookup. -/// @param corrections Output buffer (caller-owned, at least -/// `correction_length` bytes). -/// @param correction_length # of correction bytes to read. Must match the -/// decoder's declared num_observables. -/// @param reset 1 to zero the device-side correction buffer -/// after the read, 0 to leave it accumulated. __attribute__((visibility("default"))) void get_corrections(cudaq::qec::realtime::qec_realtime_session &session, std::size_t decoder_id, std::uint8_t *corrections, std::uint64_t correction_length, std::uint64_t reset); -/// @brief Send a reset RPC, clearing per-decoder device-side state -/// (correction buffer + the plugin's BP context). -/// -/// Handler is the shared DEVICE_CALL `reset_decoder_ui64`. -/// -/// @param session Realtime session. Must be `initialized()`. -/// @param decoder_id Decoder index for state lookup. __attribute__((visibility("default"))) void reset_decoder(cudaq::qec::realtime::qec_realtime_session &session, std::size_t decoder_id); -//============================================================================== -// Spin/timeout knobs (visible so tests can shorten timeouts for negative -// fixtures without touching the production defaults). -//============================================================================== - -/// @brief Max time (ms) `AcquireSlot` will spin waiting for a free slot -/// before throwing. -constexpr int kAcquireSlotTimeoutMs = 5000; - -/// @brief Max time (ms) `WaitForResponse` will spin waiting for -/// `RPCResponse::magic` before throwing. -constexpr int kResponseTimeoutMs = 5000; - } // namespace cudaq::qec::decoding::rpc_producer diff --git a/libs/qec/lib/realtime/simulation-cqr/simulation_cqr_device.cpp b/libs/qec/lib/realtime/simulation-cqr/simulation_cqr_device.cpp index 69bf7ebaa..0b5aed8fc 100644 --- a/libs/qec/lib/realtime/simulation-cqr/simulation_cqr_device.cpp +++ b/libs/qec/lib/realtime/simulation-cqr/simulation_cqr_device.cpp @@ -16,13 +16,6 @@ // fnv1a_32(), so these names are what the host-dispatch service // (decoding_server_cqr.cpp) registers. // -// The wrappers also perform the spec's on-wire rewrite themselves (until the -// device_call lowering can): syndrome/correction bits cross the wire -// bit-packed (LSB-first) in a std::vector placed last, with the -// explicit num_syndromes / return_size scalars of the spec's -// enqueue_syndromes / get_corrections wire layouts. The stdvec -// serialization's uint64 byte-count prefix is the spec's ARRAY_UINT8 prefix. -// // This translation unit is compiled with -frealtime-lowering (see the // cudaq-qec-realtime-decoding-simulation-cqr library), which rewrites each // device_call to the realtime frame ABI and dispatches it by function_id @@ -38,13 +31,11 @@ #include "cudaq.h" #include "cudaq/qec/realtime/decoding.h" -// device_call targets, in the spec's on-wire argument order (variable-length -// byte vector last). Named to match the spec function_ids. Defined (hidden) in -// simulation_cqr_host.cpp. +// device_call targets. Named to match the spec function_ids. Defined +// (hidden) in simulation_cqr_host.cpp. extern "C" { -void enqueue_syndromes(std::uint64_t decoder_id, std::uint64_t counter, - std::uint64_t syndrome_mapping_id, - const std::vector &syndrome_bits); +void enqueue_syndromes(std::uint64_t decoder_id, + const std::vector &syndromes, std::uint64_t tag); void get_corrections(std::uint64_t decoder_id, std::vector &corrections, bool reset); @@ -58,39 +49,20 @@ __qpu__ void enqueue_syndromes(std::uint64_t decoder_id, const std::vector &syndromes, std::uint64_t tag) { - // No syndrome mapping table yet; id 0 is the identity mapping. - constexpr std::uint64_t kSyndromeMappingId = 0; - // Pass the syndrome bits as a std::vector; the realtime device_call - // lowering bit-packs it LSB-first into a CUDAQ_TYPE_BIT_PACKED payload (cudaq - // PR 4816) and serializes its length as an element-count prefix. That prefix - // IS num_syndromes, so no separate num_syndromes argument is passed. - std::size_t num_bits = syndromes.size(); - std::vector bits(num_bits); - for (std::size_t i = 0; i < num_bits; ++i) - bits[i] = syndromes[i]; - cudaq::device_call(::enqueue_syndromes, decoder_id, tag, kSyndromeMappingId, - bits); + // Discriminate the measurement handles to bits before crossing the boundary. + cudaq::device_call(::enqueue_syndromes, decoder_id, + cudaq::to_bools(syndromes), tag); } __qpu__ void enqueue_syndromes_test(std::uint64_t decoder_id, const std::vector &syndromes, std::uint64_t tag) { - constexpr std::uint64_t kSyndromeMappingId = 0; - // syndromes is already a std::vector; the lowering bit-packs it and - // serializes its length as the element-count prefix (== num_syndromes). - cudaq::device_call(::enqueue_syndromes, decoder_id, tag, kSyndromeMappingId, - syndromes); + cudaq::device_call(::enqueue_syndromes, decoder_id, syndromes, tag); } __qpu__ std::vector get_corrections(std::uint64_t decoder_id, std::uint64_t return_size, bool reset) { - // The OUT corrections vector is a std::vector: the realtime lowering - // serializes its length as an element (bit) count and unpacks the BIT_PACKED - // response straight back into it (cudaq PR 4816). That length IS the return - // size, so no separate return_size argument is passed (matches - // simulation_device.cpp). reset is a trailing bool, so the request payload - // is 17 bytes with no trailing padding. std::vector result(return_size); cudaq::device_call(::get_corrections, decoder_id, result, reset); return result; diff --git a/libs/qec/lib/realtime/simulation-cqr/simulation_cqr_host.cpp b/libs/qec/lib/realtime/simulation-cqr/simulation_cqr_host.cpp index ee9f2aad8..cc0c829a7 100644 --- a/libs/qec/lib/realtime/simulation-cqr/simulation_cqr_host.cpp +++ b/libs/qec/lib/realtime/simulation-cqr/simulation_cqr_host.cpp @@ -17,11 +17,6 @@ // function_id to the service in decoding_server_cqr.cpp and never calls // these). // -// The wire arguments are the spec's on-wire form (bit-packed byte vector -// last, explicit bit-count scalars); these trampolines unpack to the -// byte-per-bool buffers the host:: decoding API takes, mirroring what the -// host-dispatch service does on the server side. -// // They are given HIDDEN visibility: the device stubs resolve them within this // library, but the deliberately generic names (enqueue_syndromes, ...) are NOT // exported, so they cannot collide with symbols in other libraries. @@ -29,52 +24,29 @@ #include "../realtime_decoding.h" #include -#include namespace { -// nvq++ lowers a std::vector (or std::vector) device_call -// argument to a {pointer, length} span; the trampolines must accept that -// layout (same trick as the simulation backend's boolean_span). -struct byte_span { +// nvq++ lowers a std::vector device_call argument to a {pointer, length} +// span; the trampolines must accept that layout (same trick as the simulation +// backend's boolean_span). +struct boolean_span { std::uint8_t *buffer; std::uint64_t length; }; } // namespace extern "C" __attribute__((visibility("hidden"))) void -enqueue_syndromes(std::uint64_t decoder_id, std::uint64_t counter, - std::uint64_t syndrome_mapping_id, byte_span syndrome_bits) { - // No syndrome mapping table yet: syndrome_mapping_id 0 is the identity - // mapping. syndrome_bits is a std::vector span: `length` is the logical - // bit count (== num_syndromes) and `buffer` points at the LSB-first - // bit-packed bytes. - (void)syndrome_mapping_id; - const std::uint64_t num_syndromes = syndrome_bits.length; - std::vector bits(num_syndromes); - for (std::uint64_t i = 0; i < num_syndromes; ++i) - bits[i] = (syndrome_bits.buffer[i / 8] >> (i % 8)) & 1; // LSB-first - cudaq::qec::decoding::host::enqueue_syndromes(decoder_id, bits.data(), - num_syndromes, counter); +enqueue_syndromes(std::uint64_t decoder_id, boolean_span syndromes, + std::uint64_t tag) { + cudaq::qec::decoding::host::enqueue_syndromes(decoder_id, syndromes.buffer, + syndromes.length, tag); } extern "C" __attribute__((visibility("hidden"))) void -get_corrections(std::uint64_t decoder_id, byte_span corrections, bool reset) { - // corrections is a std::vector span: `length` is the logical bit count - // (the return size) and `buffer` points at the LSB-first bit-packed bytes. - const std::uint64_t return_size = corrections.length; - std::vector bits(return_size); - cudaq::qec::decoding::host::get_corrections(decoder_id, bits.data(), - return_size, reset); - std::uint64_t num_bytes = (return_size + 7) / 8; - for (std::uint64_t byte = 0; byte < num_bytes; ++byte) { - std::uint8_t value = 0; - for (std::uint64_t bit = 0; bit < 8; ++bit) { - std::uint64_t index = byte * 8 + bit; - if (index < return_size && bits[index]) - value |= static_cast(1u << bit); // LSB-first - } - corrections.buffer[byte] = value; - } +get_corrections(std::uint64_t decoder_id, boolean_span corrections, + bool reset) { + cudaq::qec::decoding::host::get_corrections(decoder_id, corrections.buffer, + corrections.length, reset); } extern "C" __attribute__((visibility("hidden"))) void diff --git a/libs/qec/lib/realtime/simulation/CMakeLists.txt b/libs/qec/lib/realtime/simulation/CMakeLists.txt index 0aab2e498..4cc5331ef 100644 --- a/libs/qec/lib/realtime/simulation/CMakeLists.txt +++ b/libs/qec/lib/realtime/simulation/CMakeLists.txt @@ -25,10 +25,7 @@ target_include_directories(cudaq-qec-realtime-decoding-simulation $ ) -# PRIVATE, not PUBLIC: PUBLIC propagates --exclude-libs,ALL to every consumer, -# including test executables that absorb libcudaq-realtime-dispatch.a and -# dlsym() its CUDAQ_REALTIME_DISPATCH_API symbols at runtime. -target_link_options(cudaq-qec-realtime-decoding-simulation PRIVATE +target_link_options(cudaq-qec-realtime-decoding-simulation PUBLIC $<$:-Wl,--exclude-libs,ALL> ) diff --git a/libs/qec/python/CMakeLists.txt b/libs/qec/python/CMakeLists.txt index 4a3a1ced9..65f76c44c 100644 --- a/libs/qec/python/CMakeLists.txt +++ b/libs/qec/python/CMakeLists.txt @@ -44,7 +44,6 @@ target_link_libraries(${MODULE_NAME} cudaq-qec-decoders cudaq-qec-realtime-decoding cudaq::cudaq-python-interop - cudaq::cudaq-mlir-runtime ) find_package(CUDAToolkit REQUIRED) diff --git a/libs/qec/python/bindings/py_decoding_config.cpp b/libs/qec/python/bindings/py_decoding_config.cpp index 73e6e58ed..98a4f9aa8 100644 --- a/libs/qec/python/bindings/py_decoding_config.cpp +++ b/libs/qec/python/bindings/py_decoding_config.cpp @@ -9,143 +9,18 @@ #include "py_decoding_config.h" #include "type_casters.h" -#include "cudaq/qec/decoder_config_schema.h" #include "cudaq/qec/realtime/decoding_config.h" #include #include #include #include +#include #include -#include +#include namespace nb = nanobind; namespace cudaq::qec::decoding::config { - -namespace { - -template -T cast_param(const nb::object &value, const std::string &key, - const std::string &schema_name, const char *kind_name) { - try { - return nb::cast(value); - } catch (...) { - throw std::runtime_error("Parameter '" + key + "' of '" + schema_name + - "' expects a " + kind_name + " value."); - } -} - -// Convert a Python dict to the canonical storage types the decoder's -// registered schema declares (int32 params admit negative ints, f64 params -// admit Python ints, ...). The generic heterogeneous_map caster stores every -// Python int as std::size_t, which rejects negatives at assignment and makes -// f64 params unreadable at YAML emission / decoder construction. -cudaqx::heterogeneous_map -schema_typed_map_from_dict(const decoder_schema &schema, nb::dict dict) { - cudaqx::heterogeneous_map map; - nb::dict residual; - for (auto item : dict) { - const std::string key = nb::cast(item.first); - nb::object value = nb::borrow(item.second); - const param_spec *spec = nullptr; - for (const auto &candidate : schema.params) { - if (candidate.key == key) { - spec = &candidate; - break; - } - } - if (!spec) { - // Unknown keys keep the generic conversion so validate_custom_args and - // emission diagnostics can still name them. - residual[item.first] = item.second; - continue; - } - switch (spec->kind) { - case param_kind::boolean: - map.insert(key, cast_param(value, key, schema.name, "boolean")); - break; - case param_kind::int32: - map.insert(key, cast_param(value, key, schema.name, "32-bit int")); - break; - case param_kind::uint64: - map.insert(key, cast_param(value, key, schema.name, - "non-negative int")); - break; - case param_kind::f64: - map.insert(key, cast_param(value, key, schema.name, "float")); - break; - case param_kind::string: - map.insert(key, - cast_param(value, key, schema.name, "string")); - break; - case param_kind::f64_vec: - map.insert(key, cast_param>(value, key, schema.name, - "list-of-float")); - break; - case param_kind::f64_matrix: - map.insert(key, cast_param>>( - value, key, schema.name, "list-of-list-of-float")); - break; - case param_kind::subschema: { - const auto *nested = find_decoder_schema(spec->subschema); - if (nested && nb::isinstance(value)) - map.insert(key, schema_typed_map_from_dict(*nested, - nb::cast(value))); - else - map.insert(key, cast_param( - value, key, schema.name, "dict")); - break; - } - case param_kind::discriminated: { - const decoder_schema *nested = nullptr; - if (dict.contains(spec->discriminator.c_str())) { - nb::object discriminator = dict[spec->discriminator.c_str()]; - if (nb::isinstance(discriminator)) - nested = find_decoder_schema(nb::cast(discriminator)); - } - if (nested && nb::isinstance(value)) - map.insert(key, schema_typed_map_from_dict(*nested, - nb::cast(value))); - else - map.insert(key, cast_param( - value, key, schema.name, "dict")); - break; - } - } - } - if (nb::len(residual) > 0) { - auto generic = nb::cast(nb::object(residual)); - for (const auto &kv : generic) - map.insert(kv.first, kv.second); - } - return map; -} - -cudaqx::heterogeneous_map -custom_args_map_from_python(const std::string &decoder_type, nb::object value) { - std::string schema_name = decoder_type; - if (!nb::isinstance(value) && - nb::hasattr(value, "to_heterogeneous_map")) { - // Deprecated typed-config path (the cudaq_qec._compat shims and the - // pre-schema classes they replace): the object reduces itself to a dict - // of the explicitly-set parameters, and carries the decoder schema name - // to convert that dict with, so conversion does not depend on - // decoder_config.type having been assigned first. - if (nb::hasattr(value, "_schema_name")) { - nb::object schema_attr = value.attr("_schema_name"); - if (nb::isinstance(schema_attr)) - schema_name = nb::cast(schema_attr); - } - value = value.attr("to_heterogeneous_map")(); - } - if (nb::isinstance(value)) - if (const auto *schema = find_decoder_schema(schema_name)) - return schema_typed_map_from_dict(*schema, nb::cast(value)); - return nb::cast(value); -} - -} // namespace - void bindDecodingConfig(nb::module_ &mod) { auto qecmod = nb::hasattr(mod, "qecrt") ? nb::cast(mod.attr("qecrt")) @@ -154,6 +29,251 @@ void bindDecodingConfig(nb::module_ &mod) { auto mod_cfg = qecmod.def_submodule("config", "Realtime decoding configuration"); + // Allow Python None to clear std::optional fields. + const auto setter_accepts_none = nb::for_setter(nb::arg("value").none()); + + // srelay_bp_config + nb::class_(mod_cfg, "srelay_bp_config", + "Relay-BP decoder configuration.") + .def(nb::init<>()) + .def( + "__init__", + [](config::srelay_bp_config &self, + const cudaqx::heterogeneous_map &map) { + new (&self) + srelay_bp_config(srelay_bp_config::from_heterogeneous_map(map)); + }, + nb::arg("map")) + .def_rw("pre_iter", &srelay_bp_config::pre_iter) + .def_rw("num_sets", &srelay_bp_config::num_sets) + .def_rw("stopping_criterion", &srelay_bp_config::stopping_criterion) + .def_rw("stop_nconv", &srelay_bp_config::stop_nconv) + .def("to_heterogeneous_map", &srelay_bp_config::to_heterogeneous_map, + nb::rv_policy::move) + .def_static("from_heterogeneous_map", + &srelay_bp_config::from_heterogeneous_map, nb::arg("map")); + + // nv_qldpc_decoder_config + nb::class_( + mod_cfg, "nv_qldpc_decoder_config", "Optional decoder custom args.") + .def(nb::init<>()) + .def( + "__init__", + [](config::nv_qldpc_decoder_config &self, + const cudaqx::heterogeneous_map &map) { + new (&self) nv_qldpc_decoder_config( + nv_qldpc_decoder_config::from_heterogeneous_map(map)); + }, + nb::arg("map")) + .def_rw("use_sparsity", &nv_qldpc_decoder_config::use_sparsity) + .def_rw("error_rate", &nv_qldpc_decoder_config::error_rate) + .def_rw("error_rate_vec", &nv_qldpc_decoder_config::error_rate_vec) + .def_rw("max_iterations", &nv_qldpc_decoder_config::max_iterations) + .def_rw("n_threads", &nv_qldpc_decoder_config::n_threads) + .def_rw("use_osd", &nv_qldpc_decoder_config::use_osd) + .def_rw("osd_method", &nv_qldpc_decoder_config::osd_method) + .def_rw("osd_order", &nv_qldpc_decoder_config::osd_order) + .def_rw("bp_batch_size", &nv_qldpc_decoder_config::bp_batch_size) + .def_rw("osd_batch_size", &nv_qldpc_decoder_config::osd_batch_size) + .def_rw("iter_per_check", &nv_qldpc_decoder_config::iter_per_check) + .def_rw("clip_value", &nv_qldpc_decoder_config::clip_value) + .def_rw("bp_method", &nv_qldpc_decoder_config::bp_method) + .def_rw("scale_factor", &nv_qldpc_decoder_config::scale_factor) + .def_rw("proc_float", &nv_qldpc_decoder_config::proc_float) + .def_rw("gamma0", &nv_qldpc_decoder_config::gamma0) + .def_rw("gamma_dist", &nv_qldpc_decoder_config::gamma_dist) + .def_rw("explicit_gammas", &nv_qldpc_decoder_config::explicit_gammas) + .def_rw("srelay_config", &nv_qldpc_decoder_config::srelay_config) + .def_rw("bp_seed", &nv_qldpc_decoder_config::bp_seed) + .def_rw("composition", &nv_qldpc_decoder_config::composition) + .def("to_heterogeneous_map", + &nv_qldpc_decoder_config::to_heterogeneous_map, nb::rv_policy::move) + .def_static("from_heterogeneous_map", + &nv_qldpc_decoder_config::from_heterogeneous_map, + nb::arg("map")); + + // multi_error_lut_config + nb::class_(mod_cfg, "multi_error_lut_config", + "Optional decoder custom args.") + .def(nb::init<>()) + .def( + "__init__", + [](config::multi_error_lut_config &self, + const cudaqx::heterogeneous_map &map) { + new (&self) multi_error_lut_config( + multi_error_lut_config::from_heterogeneous_map(map)); + }, + nb::arg("map")) + .def_rw("lut_error_depth", &multi_error_lut_config::lut_error_depth) + .def("to_heterogeneous_map", + &multi_error_lut_config::to_heterogeneous_map, nb::rv_policy::move) + .def_static("from_heterogeneous_map", + &multi_error_lut_config::from_heterogeneous_map, + nb::arg("map")); + + // trt_decoder_config + nb::class_(mod_cfg, "trt_decoder_config", + "TensorRT decoder configuration.") + .def(nb::init<>()) + .def( + "__init__", + [](config::trt_decoder_config &self, + const cudaqx::heterogeneous_map &map) { + new (&self) trt_decoder_config( + trt_decoder_config::from_heterogeneous_map(map)); + }, + nb::arg("map")) + .def_rw("onnx_load_path", &trt_decoder_config::onnx_load_path, + setter_accepts_none) + .def_rw("engine_load_path", &trt_decoder_config::engine_load_path, + setter_accepts_none) + .def_rw("engine_save_path", &trt_decoder_config::engine_save_path, + setter_accepts_none) + .def_rw("precision", &trt_decoder_config::precision, setter_accepts_none) + .def_rw("memory_workspace", &trt_decoder_config::memory_workspace, + setter_accepts_none) + .def_rw("batch_size", &trt_decoder_config::batch_size, + setter_accepts_none) + .def_rw("use_cuda_graph", &trt_decoder_config::use_cuda_graph, + setter_accepts_none) + .def_rw("global_decoder", &trt_decoder_config::global_decoder, + setter_accepts_none) + .def_prop_rw( + "global_decoder_params", + [](const trt_decoder_config &self) -> nb::object { + if (std::holds_alternative( + self.global_decoder_params)) { + return nb::cast( + std::get(self.global_decoder_params)); + } + if (std::holds_alternative( + self.global_decoder_params)) { + return nb::cast( + std::get(self.global_decoder_params)); + } + return nb::none(); + }, + [](trt_decoder_config &self, nb::object value) { + if (value.is_none()) { + self.global_decoder_params = std::monostate(); + } else if (nb::isinstance(value)) { + self.global_decoder_params = nb::cast(value); + } else if (nb::isinstance(value)) { + self.global_decoder_params = nb::cast(value); + } else { + throw nb::type_error( + "global_decoder_params must be pymatching_config, " + "chromobius_config, or None."); + } + }, + setter_accepts_none) + .def("to_heterogeneous_map", &trt_decoder_config::to_heterogeneous_map, + nb::rv_policy::move) + .def_static("from_heterogeneous_map", + &trt_decoder_config::from_heterogeneous_map, nb::arg("map")); + + // pymatching_config + nb::class_(mod_cfg, "pymatching_config", + "PyMatching decoder configuration.") + .def(nb::init<>()) + .def( + "__init__", + [](config::pymatching_config &self, + const cudaqx::heterogeneous_map &map) { + new (&self) pymatching_config( + pymatching_config::from_heterogeneous_map(map)); + }, + nb::arg("map")) + .def_rw("error_rate_vec", &pymatching_config::error_rate_vec) + .def_rw("merge_strategy", &pymatching_config::merge_strategy) + .def("to_heterogeneous_map", &pymatching_config::to_heterogeneous_map, + nb::rv_policy::move) + .def_static("from_heterogeneous_map", + &pymatching_config::from_heterogeneous_map, nb::arg("map")); + + // chromobius_config + nb::class_(mod_cfg, "chromobius_config", + "Chromobius decoder configuration.") + .def(nb::init<>()) + .def( + "__init__", + [](config::chromobius_config &self, + const cudaqx::heterogeneous_map &map) { + new (&self) chromobius_config( + chromobius_config::from_heterogeneous_map(map)); + }, + nb::arg("map")) + .def_rw("drop_mobius_errors_involving_remnant_errors", + &chromobius_config::drop_mobius_errors_involving_remnant_errors, + setter_accepts_none) + .def_rw("ignore_decomposition_failures", + &chromobius_config::ignore_decomposition_failures, + setter_accepts_none) + .def_rw("include_coords_in_mobius_dem", + &chromobius_config::include_coords_in_mobius_dem, + setter_accepts_none) + .def_rw("return_weight", &chromobius_config::return_weight, + setter_accepts_none) + .def_rw("write_mobius_match_to_stderr", + &chromobius_config::write_mobius_match_to_stderr, + setter_accepts_none) + .def("to_heterogeneous_map", &chromobius_config::to_heterogeneous_map, + nb::rv_policy::move) + .def_static("from_heterogeneous_map", + &chromobius_config::from_heterogeneous_map, nb::arg("map")); + + // single_error_lut_config + nb::class_( + mod_cfg, "single_error_lut_config", + "Single error LUT decoder configuration.") + .def(nb::init<>()) + .def( + "__init__", + [](config::single_error_lut_config &self, + const cudaqx::heterogeneous_map &map) { + new (&self) single_error_lut_config( + single_error_lut_config::from_heterogeneous_map(map)); + }, + nb::arg("map")) + .def("to_heterogeneous_map", + &single_error_lut_config::to_heterogeneous_map, nb::rv_policy::move) + .def_static("from_heterogeneous_map", + &single_error_lut_config::from_heterogeneous_map, + nb::arg("map")); + + // sliding_window_config + nb::class_( + mod_cfg, "sliding_window_config", "Sliding window decoder configuration.") + .def(nb::init<>()) + .def( + "__init__", + [](config::sliding_window_config &self, + const cudaqx::heterogeneous_map &map) { + new (&self) sliding_window_config( + sliding_window_config::from_heterogeneous_map(map)); + }, + nb::arg("map")) + .def_rw("window_size", &sliding_window_config::window_size) + .def_rw("step_size", &sliding_window_config::step_size) + .def_rw("num_syndromes_per_round", + &sliding_window_config::num_syndromes_per_round) + .def_rw("straddle_start_round", + &sliding_window_config::straddle_start_round) + .def_rw("straddle_end_round", &sliding_window_config::straddle_end_round) + .def_rw("error_rate_vec", &sliding_window_config::error_rate_vec) + .def_rw("inner_decoder_name", &sliding_window_config::inner_decoder_name) + .def_rw("single_error_lut_params", + &sliding_window_config::single_error_lut_params) + .def_rw("multi_error_lut_params", + &sliding_window_config::multi_error_lut_params) + .def_rw("nv_qldpc_decoder_params", + &sliding_window_config::nv_qldpc_decoder_params) + .def("to_heterogeneous_map", &sliding_window_config::to_heterogeneous_map, + nb::rv_policy::move) + .def_static("from_heterogeneous_map", + &sliding_window_config::from_heterogeneous_map, + nb::arg("map")); + // decoder_config nb::class_(mod_cfg, "decoder_config") .def(nb::init<>()) @@ -165,36 +285,22 @@ void bindDecodingConfig(nb::module_ &mod) { .def_rw("H_sparse", &decoder_config::H_sparse) .def_rw("O_sparse", &decoder_config::O_sparse) .def_rw("D_sparse", &decoder_config::D_sparse) - .def_prop_rw( - "decoder_custom_args", - [](const decoder_config &self) -> nb::object { - return nb::cast(self.decoder_custom_args.map()); - }, - [](decoder_config &self, nb::object value) { - self.decoder_custom_args = - custom_args_map_from_python(self.type, value); - }, - "The decoder's parameter dict. Keys are governed by the parameter " - "schema the decoder registered (see decoder_param_schema()); set " - "`type` before assigning so values are converted to the schema's " - "declared types. Reading returns a copy: mutate a local dict and " - "assign it back rather than mutating the returned value in place.") + .def_rw("decoder_custom_args", &decoder_config::decoder_custom_args) .def( "set_decoder_custom_args", - [](config::decoder_config &self, nb::object custom_args) { - self.decoder_custom_args = - custom_args_map_from_python(self.type, custom_args); + [](config::decoder_config &self, nb::object decoder_config) { + if (nb::hasattr(decoder_config, "to_heterogeneous_map")) { + nb::object hm_object = + decoder_config.attr("to_heterogeneous_map")(); + cudaqx::heterogeneous_map hm = + nb::cast(hm_object); + self.set_decoder_custom_args_from_heterogeneous_map(hm); + return; + } + throw nb::type_error("set_decoder_custom_args expects an object " + "with to_heterogeneous_map()."); }, - nb::arg("custom_args"), - "Set the decoder parameter dict for this decoder (equivalent to " - "assigning decoder_custom_args; set `type` first).") - .def("validate_custom_args", &decoder_config::validate_custom_args, - "Validate decoder_custom_args against the parameter schema " - "registered for this decoder type: unknown keys, missing required " - "keys, and the schema's own validation hook. Raises RuntimeError " - "on the first violation. YAML parsing applies the same checks " - "automatically; call this to vet a configuration built " - "programmatically before using it.") + nb::arg("custom_args_obj")) .def("to_yaml_str", &decoder_config::to_yaml_str, nb::arg("column_wrap") = 80) .def_static("from_yaml_str", &decoder_config::from_yaml_str, @@ -207,9 +313,6 @@ void bindDecodingConfig(nb::module_ &mod) { nb::class_(mod_cfg, "multi_decoder_config") .def(nb::init<>()) .def_rw("decoders", &multi_decoder_config::decoders) - .def("validate_custom_args", &multi_decoder_config::validate_custom_args, - "Validate every decoder's custom args against its registered " - "parameter schema (see decoder_config.validate_custom_args).") .def("to_yaml_str", &multi_decoder_config::to_yaml_str, nb::arg("column_wrap") = 80) .def_static("from_yaml_str", &multi_decoder_config::from_yaml_str, @@ -229,64 +332,5 @@ void bindDecodingConfig(nb::module_ &mod) { "Configure decoders from a YAML string; returns int status."); mod_cfg.def("finalize_decoders", &finalize_decoders, "Finalize decoder resources."); - mod_cfg.def( - "decoder_param_schema", - [](const std::string &name) -> nb::object { - const auto *schema = find_decoder_schema(name); - if (!schema) - return nb::none(); - auto kind_name = [](param_kind kind) -> const char * { - switch (kind) { - case param_kind::boolean: - return "bool"; - case param_kind::int32: - return "int32"; - case param_kind::uint64: - return "uint64"; - case param_kind::f64: - return "float64"; - case param_kind::string: - return "string"; - case param_kind::f64_vec: - return "float64_vec"; - case param_kind::f64_matrix: - return "float64_matrix"; - case param_kind::subschema: - return "subschema"; - case param_kind::discriminated: - return "discriminated"; - } - return "unknown"; - }; - nb::list params; - for (const auto &spec : schema->params) { - nb::dict entry; - entry["key"] = spec.key; - entry["kind"] = kind_name(spec.kind); - entry["required"] = spec.required; - if (!spec.subschema.empty()) - entry["subschema"] = spec.subschema; - if (!spec.discriminator.empty()) - entry["discriminator"] = spec.discriminator; - params.append(entry); - } - return params; - }, - nb::arg("decoder_name"), - "Return the registered custom-args parameter schema for a decoder " - "(list of parameter descriptors), or None if the decoder has not " - "registered one."); - mod_cfg.def("registered_decoder_schemas", ®istered_decoder_schema_names, - "Names of all decoders (and nested sections) with registered " - "custom-args parameter schemas."); - mod_cfg.def( - "decoder_config_json_schema", &decoder_config_json_schema, - "Return a JSON Schema (draft 2020-12) document, as a string, that " - "validates multi_decoder_config YAML files. Generated from the decoder " - "parameter schemas registered in this installation (including loaded " - "third-party decoder plugins), for use with standard tools such as " - "check-jsonschema or the python jsonschema package. Schema validate " - "hooks are not representable in JSON Schema, so a passing document may " - "still be rejected when parsed."); } } // namespace cudaq::qec::decoding::config diff --git a/libs/qec/python/bindings/type_casters.h b/libs/qec/python/bindings/type_casters.h index fbe3f3c26..72c98df46 100644 --- a/libs/qec/python/bindings/type_casters.h +++ b/libs/qec/python/bindings/type_casters.h @@ -14,6 +14,7 @@ #include "cuda-qx/core/heterogeneous_map.h" #include "cuda-qx/core/kwargs_utils.h" #include "cuda-qx/core/tensor.h" +#include "cudaq/qec/realtime/decoding_config.h" #include #include #include @@ -179,6 +180,53 @@ struct type_caster { std::any_cast(&val)) { // Recursively convert nested heterogeneous_map result[key.c_str()] = nb::cast(*hetMap); + } else if (auto *srelay_cfg = std::any_cast< + cudaq::qec::decoding::config::srelay_bp_config>(&val)) { + result[key.c_str()] = nb::cast(srelay_cfg->to_heterogeneous_map()); + } else if (auto *nv_cfg = std::any_cast< + cudaq::qec::decoding::config::nv_qldpc_decoder_config>( + &val)) { + result[key.c_str()] = nb::cast(nv_cfg->to_heterogeneous_map()); + } else if (auto *multi_cfg = std::any_cast< + cudaq::qec::decoding::config::multi_error_lut_config>( + &val)) { + result[key.c_str()] = nb::cast(multi_cfg->to_heterogeneous_map()); + } else if (auto *single_cfg = std::any_cast< + cudaq::qec::decoding::config::single_error_lut_config>( + &val)) { + result[key.c_str()] = nb::cast(single_cfg->to_heterogeneous_map()); + } else if (auto *global_cfg = std::any_cast< + cudaq::qec::decoding::config::global_decoder_config>( + &val)) { + if (std::holds_alternative(*global_cfg)) { + // Omit the key for monostate, matching + // trt_decoder_config::to_heterogeneous_map(): an unset global + // decoder serializes as absent, not as a null value. + } else if (std::holds_alternative< + cudaq::qec::decoding::config::pymatching_config>( + *global_cfg)) { + result[key.c_str()] = nb::cast( + std::get( + *global_cfg) + .to_heterogeneous_map()); + } else { + result[key.c_str()] = nb::cast( + std::get( + *global_cfg) + .to_heterogeneous_map()); + } + } else if (auto *pymatching_cfg = std::any_cast< + cudaq::qec::decoding::config::pymatching_config>(&val)) { + result[key.c_str()] = + nb::cast(pymatching_cfg->to_heterogeneous_map()); + } else if (auto *chromobius_cfg = std::any_cast< + cudaq::qec::decoding::config::chromobius_config>(&val)) { + result[key.c_str()] = + nb::cast(chromobius_cfg->to_heterogeneous_map()); + } else if (auto *sw_cfg = std::any_cast< + cudaq::qec::decoding::config::sliding_window_config>( + &val)) { + result[key.c_str()] = nb::cast(sw_cfg->to_heterogeneous_map()); } else { PyErr_SetString(PyExc_RuntimeError, ("Failed to cast from heterogeneous_map to " diff --git a/libs/qec/python/cudaq_qec/__init__.py b/libs/qec/python/cudaq_qec/__init__.py index 742a35d09..310785e7d 100644 --- a/libs/qec/python/cudaq_qec/__init__.py +++ b/libs/qec/python/cudaq_qec/__init__.py @@ -114,26 +114,15 @@ def checked_decode_batch(self, *args, **kwargs): multi_decoder_config = qecrt.config.multi_decoder_config decoder_config = qecrt.config.decoder_config +nv_qldpc_decoder_config = qecrt.config.nv_qldpc_decoder_config +multi_error_lut_config = qecrt.config.multi_error_lut_config +trt_decoder_config = qecrt.config.trt_decoder_config +pymatching_config = qecrt.config.pymatching_config +chromobius_config = qecrt.config.chromobius_config configure_decoders_from_file = qecrt.config.configure_decoders_from_file configure_decoders_from_str = qecrt.config.configure_decoders_from_str finalize_decoders = qecrt.config.finalize_decoders configure_decoders = qecrt.config.configure_decoders -decoder_param_schema = qecrt.config.decoder_param_schema -registered_decoder_schemas = qecrt.config.registered_decoder_schemas -decoder_config_json_schema = qecrt.config.decoder_config_json_schema - -# Deprecated typed decoder-config classes, kept for backward compatibility -# with the pre-schema API. They warn on construction; new code should assign -# plain dicts to decoder_config.decoder_custom_args instead. -from . import _compat -from ._compat import (nv_qldpc_decoder_config, multi_error_lut_config, - trt_decoder_config, pymatching_config, chromobius_config) - -for _compat_cls_name in _compat.__all__: - # The classes used to live in the compiled config submodule; keep that - # spelling working too (e.g. qec.qecrt.config.sliding_window_config). - setattr(qecrt.config, _compat_cls_name, getattr(_compat, _compat_cls_name)) -del _compat_cls_name stabilizer_grid = qecrt.stabilizer_grid role_to_str = qecrt.role_to_str diff --git a/libs/qec/python/cudaq_qec/_compat.py b/libs/qec/python/cudaq_qec/_compat.py deleted file mode 100644 index 761f07796..000000000 --- a/libs/qec/python/cudaq_qec/_compat.py +++ /dev/null @@ -1,321 +0,0 @@ -# ============================================================================ # -# Copyright (c) 2024 - 2026 NVIDIA Corporation & Affiliates. # -# All rights reserved. # -# # -# This source code and the accompanying materials are made available under # -# the terms of the Apache License 2.0 which accompanies this distribution. # -# ============================================================================ # -"""Deprecated typed decoder-config classes. - -These classes reproduce the pre-schema typed configuration API -(``nv_qldpc_decoder_config`` and friends) on top of the schema-driven dict -interface that replaced it. Each instance is a thin wrapper around a dict of -explicitly-set parameters; assigning one to -``decoder_config.decoder_custom_args`` (or passing it to -``decoder_config.set_decoder_custom_args``) converts that dict through the -parameter schema the decoder registered, exactly as if the dict had been -assigned directly. - -New code should assign plain dicts instead:: - - dc.decoder_custom_args = {"max_iterations": 50} - -and can discover any decoder's parameters with -``cudaq_qec.decoder_param_schema(name)``. These classes emit a -``DeprecationWarning`` on construction and will be removed in a future -release. -""" - -import warnings - -__all__ = [ - "srelay_bp_config", - "nv_qldpc_decoder_config", - "single_error_lut_config", - "multi_error_lut_config", - "pymatching_config", - "chromobius_config", - "trt_decoder_config", - "sliding_window_config", -] - - -class _deprecated_typed_config: - """Base for the deprecated typed decoder-config shims. - - ``_fields`` is the exact attribute surface the pre-schema class exposed; - every field behaves like the ``std::optional`` it used to be: unset reads - as ``None``, assigning ``None`` clears it, and ``to_heterogeneous_map`` - only emits fields that were explicitly set. - """ - - # Overridden by subclasses. ``_schema_name`` names the registered decoder - # parameter schema used to convert the dict when the shim is handed to a - # decoder_config (it wins over decoder_config.type, so old code works - # regardless of assignment order). - _schema_name = None - _fields = () - - def __init__(self, map=None): - warnings.warn( - f"{type(self).__name__} is deprecated; assign a plain dict to " - "decoder_config.decoder_custom_args instead (see " - "cudaq_qec.decoder_param_schema for the accepted keys).", - DeprecationWarning, - stacklevel=2) - object.__setattr__(self, "_args", {}) - if map is not None: - self._load_map(dict(map)) - - def _load_map(self, map): - # Mirrors the old from_heterogeneous_map: known keys are read, unknown - # keys are ignored. - for key, value in map.items(): - if key in self._fields: - self._args[key] = value - - @classmethod - def from_heterogeneous_map(cls, map): - return cls(map) - - def to_heterogeneous_map(self): - """Return the explicitly-set parameters as a plain dict.""" - out = {} - for key in self._fields: - if key in self._args: - out[key] = _as_plain_value(self._args[key]) - return out - - @classmethod - def _param_kinds(cls): - # Field name -> schema kind, from the decoder's registered parameter - # schema. Lazy: the registry is populated at plugin load, and a miss - # is not cached so a schema registered later is still picked up. - if not cls.__dict__.get("_kind_cache"): - kinds = {} - try: - from . import decoder_param_schema - schema = decoder_param_schema(cls._schema_name) - if schema: - kinds = {entry["key"]: entry["kind"] for entry in schema} - except Exception: - pass - cls._kind_cache = kinds - return cls._kind_cache - - def __setattr__(self, name, value): - if name not in self._fields: - raise AttributeError( - f"'{type(self).__name__}' object has no attribute '{name}'") - if value is None: - self._args.pop(name, None) - return - kind = self._param_kinds().get(name) - if kind is not None: - _check_param_kind(type(self).__name__, name, kind, value) - self._args[name] = value - - def __getattr__(self, name): - # Only reached when normal lookup fails, i.e. for unset fields and - # genuinely unknown attributes. - if name in type(self)._fields: - return object.__getattribute__(self, "_args").get(name) - raise AttributeError( - f"'{type(self).__name__}' object has no attribute '{name}'") - - def __eq__(self, other): - return type(other) is type(self) and self._args == other._args - - def __repr__(self): - settings = ", ".join( - f"{k}={self._args[k]!r}" for k in self._fields if k in self._args) - return f"{type(self).__name__}({settings})" - - -def _as_plain_value(value): - if isinstance(value, _deprecated_typed_config): - return value.to_heterogeneous_map() - return value - - -def _is_number(value): - return isinstance(value, (int, float)) and not isinstance(value, bool) - - -def _check_param_kind(cls_name, name, kind, value): - # Reproduce the TypeErrors the old nanobind setters raised for - # clearly-wrong values. Sequence kinds only inspect list/tuple contents, - # so array-likes (e.g. numpy) pass through to the conversion layer. - def fail(expected): - raise TypeError(f"{cls_name}.{name} expects {expected}, got " - f"{type(value).__name__}") - - if kind == "bool": - if not isinstance(value, bool): - fail("a bool") - elif kind == "int32": - if not isinstance(value, int) or isinstance(value, bool): - fail("an int") - elif kind == "uint64": - if not isinstance(value, int) or isinstance(value, bool) or value < 0: - fail("a non-negative int") - elif kind == "float64": - if not _is_number(value): - fail("a float") - elif kind == "string": - if not isinstance(value, str): - fail("a str") - elif kind == "float64_vec": - if _is_number(value) or isinstance(value, (str, bytes, dict)): - fail("a list of floats") - if isinstance(value, - (list, tuple)) and not all(_is_number(x) for x in value): - fail("a list of floats") - elif kind == "float64_matrix": - if _is_number(value) or isinstance(value, (str, bytes, dict)): - fail("a list of lists of floats") - if isinstance(value, (list, tuple)) and not all( - isinstance(row, - (list, tuple)) and all(_is_number(x) - for x in row) - for row in value): - fail("a list of lists of floats") - elif kind in ("subschema", "discriminated"): - if not isinstance(value, (_deprecated_typed_config, dict)): - fail("a config object or dict") - - -class srelay_bp_config(_deprecated_typed_config): - """Deprecated: Relay-BP decoder configuration.""" - _schema_name = "srelay_bp" - _fields = ("pre_iter", "num_sets", "stopping_criterion", "stop_nconv") - - -class nv_qldpc_decoder_config(_deprecated_typed_config): - """Deprecated: nv-qldpc-decoder custom args.""" - _schema_name = "nv-qldpc-decoder" - _fields = ("use_sparsity", "error_rate", "error_rate_vec", "max_iterations", - "n_threads", "use_osd", "osd_method", "osd_order", - "bp_batch_size", "osd_batch_size", "iter_per_check", - "clip_value", "bp_method", "scale_factor", "proc_float", - "gamma0", "gamma_dist", "explicit_gammas", "srelay_config", - "bp_seed", "composition") - - def _load_map(self, map): - srelay = map.pop("srelay_config", None) - super()._load_map(map) - if srelay is not None: - self._args["srelay_config"] = srelay_bp_config(srelay) - - -class single_error_lut_config(_deprecated_typed_config): - """Deprecated: single_error_lut decoder configuration.""" - _schema_name = "single_error_lut" - _fields = () - - -class multi_error_lut_config(_deprecated_typed_config): - """Deprecated: multi_error_lut decoder configuration.""" - _schema_name = "multi_error_lut" - _fields = ("lut_error_depth",) - - -class pymatching_config(_deprecated_typed_config): - """Deprecated: pymatching decoder configuration.""" - _schema_name = "pymatching" - _fields = ("error_rate_vec", "merge_strategy") - - -class chromobius_config(_deprecated_typed_config): - """Deprecated: chromobius decoder configuration.""" - _schema_name = "chromobius" - _fields = ("drop_mobius_errors_involving_remnant_errors", - "ignore_decomposition_failures", "include_coords_in_mobius_dem", - "return_weight", "write_mobius_match_to_stderr") - - -class trt_decoder_config(_deprecated_typed_config): - """Deprecated: trt_decoder custom args.""" - _schema_name = "trt_decoder" - _fields = ("onnx_load_path", "engine_load_path", "engine_save_path", - "precision", "memory_workspace", "batch_size", "use_cuda_graph", - "global_decoder", "global_decoder_params") - - _global_decoder_classes = { - "pymatching": pymatching_config, - "chromobius": chromobius_config, - } - - def _load_map(self, map): - params = map.pop("global_decoder_params", None) - super()._load_map(map) - cls = self._global_decoder_classes.get(self._args.get("global_decoder")) - if params is not None: - if cls is None: - # Matches the old from_heterogeneous_map: this typed API only - # ever recognized pymatching/chromobius global decoders. Use - # plain dicts to configure any other registered decoder. - raise RuntimeError( - "global_decoder_params does not support global_decoder " - f"'{self._args.get('global_decoder')}'") - self._args["global_decoder_params"] = cls(params) - elif cls is not None: - # The old from_heterogeneous_map materialized a default typed - # config for a recognized global_decoder. - self._args["global_decoder_params"] = cls() - - def to_heterogeneous_map(self): - out = super().to_heterogeneous_map() - # The old struct always emitted a (possibly empty) params section for - # a recognized global_decoder. The framework's materialize_empty - # defaulting produces the same final map either way, but old code may - # inspect this map directly. - if ("global_decoder_params" not in out and - self._args.get("global_decoder") - in self._global_decoder_classes): - out["global_decoder_params"] = {} - return out - - -class sliding_window_config(_deprecated_typed_config): - """Deprecated: sliding_window decoder custom args.""" - _schema_name = "sliding_window" - _fields = ("window_size", "step_size", "num_syndromes_per_round", - "straddle_start_round", "straddle_end_round", "error_rate_vec", - "inner_decoder_name", "single_error_lut_params", - "multi_error_lut_params", "nv_qldpc_decoder_params") - - # The old typed struct had one field per supported inner decoder but - # always emitted a single "inner_decoder_params" section, taking the first - # set field in this order. - _inner_param_fields = ( - ("single_error_lut_params", "single_error_lut", - single_error_lut_config), - ("multi_error_lut_params", "multi_error_lut", multi_error_lut_config), - ("nv_qldpc_decoder_params", "nv-qldpc-decoder", - nv_qldpc_decoder_config), - ) - - def _load_map(self, map): - inner = map.pop("inner_decoder_params", None) - super()._load_map(map) - if inner is not None: - for field, decoder_name, cls in self._inner_param_fields: - if self._args.get("inner_decoder_name") == decoder_name: - self._args[field] = cls(inner) - break - - def to_heterogeneous_map(self): - out = {} - inner_fields = tuple(f for f, _, _ in self._inner_param_fields) - for key in self._fields: - if key in self._args and key not in inner_fields: - out[key] = _as_plain_value(self._args[key]) - for field, _, _ in self._inner_param_fields: - if field in self._args: - inner = _as_plain_value(self._args[field]) - # The old struct only emitted a non-empty section. - if inner: - out["inner_decoder_params"] = inner - break - return out diff --git a/libs/qec/python/tests/test_decoders_yaml.py b/libs/qec/python/tests/test_decoders_yaml.py index 474c8483f..a6e017278 100644 --- a/libs/qec/python/tests/test_decoders_yaml.py +++ b/libs/qec/python/tests/test_decoders_yaml.py @@ -91,37 +91,39 @@ def create_test_decoder_config_nv_qldpc(decoder_id): config = create_test_empty_decoder_config(decoder_id) config.type = "nv-qldpc-decoder" - # Create NV-QLDPC decoder configuration (a parameter dict; keys are - # governed by the decoder's registered schema) - config.decoder_custom_args = { - "use_sparsity": True, - "max_iterations": 50, - "use_osd": True, - "osd_order": 60, - "osd_method": 3, - "error_rate_vec": [0.1] * config.block_size, - "n_threads": 128, - "bp_batch_size": 1, - "osd_batch_size": 16, - "iter_per_check": 2, - "clip_value": 10.0, - "bp_method": 3, - "scale_factor": 1.0, - "proc_float": "fp64", - # Relay-BP configuration - "gamma0": 0.0, - "gamma_dist": [0.1, 0.2], - "srelay_config": { - "pre_iter": 5, - "num_sets": 10, - "stopping_criterion": "NConv", - "stop_nconv": 10, - }, - # explicit_gammas must have num_sets rows (10 in this case) - "explicit_gammas": [[0.1] * config.block_size for _ in range(10)], - "bp_seed": 42, - "composition": 1, - } + # Create NV-QLDPC decoder configuration + nv_config = qec.nv_qldpc_decoder_config() + nv_config.use_sparsity = True + nv_config.max_iterations = 50 + nv_config.use_osd = True + nv_config.osd_order = 60 + nv_config.osd_method = 3 + nv_config.error_rate_vec = [0.1] * config.block_size + + nv_config.n_threads = 128 + nv_config.bp_batch_size = 1 + nv_config.osd_batch_size = 16 + nv_config.iter_per_check = 2 + nv_config.clip_value = 10.0 + nv_config.bp_method = 3 + nv_config.scale_factor = 1.0 + nv_config.proc_float = "fp64" + + # Relay-BP configuration + nv_config.gamma0 = 0.0 + nv_config.gamma_dist = [0.1, 0.2] + nv_config.srelay_config = qec.qecrt.config.srelay_bp_config() + nv_config.srelay_config.pre_iter = 5 + nv_config.srelay_config.num_sets = 10 + nv_config.srelay_config.stopping_criterion = "NConv" + nv_config.srelay_config.stop_nconv = 10 + # explicit_gammas must have num_sets rows (10 in this case) + nv_config.explicit_gammas = [[0.1] * config.block_size for _ in range(10)] + nv_config.bp_seed = 42 + nv_config.composition = 1 + + # Set the custom args + config.set_decoder_custom_args(nv_config) return config @@ -163,7 +165,9 @@ def test_multi_lut_decoder(): config = create_test_empty_decoder_config(0) config.type = "multi_error_lut" - config.decoder_custom_args = {"lut_error_depth": 2} + lut_config = qec.multi_error_lut_config() + lut_config.lut_error_depth = 2 + config.set_decoder_custom_args(lut_config) multi_config.decoders = [config] @@ -179,7 +183,8 @@ def test_single_lut_decoder(): config = create_test_empty_decoder_config(0) config.type = "single_error_lut" - config.decoder_custom_args = {} + single_lut_config = qec.qecrt.config.single_error_lut_config() + config.set_decoder_custom_args(single_lut_config) multi_config.decoders = [config] @@ -223,58 +228,26 @@ def test_sliding_window_decoder(): config.D_sparse = qec.generate_timelike_sparse_detector_matrix( config.syndrome_size, 2, include_first_round=False) - # Sliding window config. inner_decoder_params is validated against the - # schema registered under inner_decoder_name. - config.decoder_custom_args = { - "window_size": 1, - "step_size": 1, - "num_syndromes_per_round": n_syndromes_per_round, - "straddle_start_round": False, - "straddle_end_round": True, - "error_rate_vec": [0.1] * config.block_size, - "inner_decoder_name": "multi_error_lut", - "inner_decoder_params": { - "lut_error_depth": 2 - }, - } + # Sliding window config + sw_config = qec.qecrt.config.sliding_window_config() + sw_config.window_size = 1 + sw_config.step_size = 1 + sw_config.num_syndromes_per_round = n_syndromes_per_round + sw_config.straddle_start_round = False + sw_config.straddle_end_round = True + sw_config.error_rate_vec = [0.1] * config.block_size - multi_config.decoders = [config] - - check_decoder_yaml_roundtrip(multi_config) - check_decoder_creation(multi_config) - - -def test_sliding_window_boundary_syndromes_roundtrip(): - """ - Test that a sliding_window's num_boundary_syndromes parameter survives a - YAML round trip. This is serialization-only (the boundary-layout decoding - behavior is exercised by the direct-decoder tests in test_sliding_window). - """ - multi_config = qec.multi_decoder_config() - config = create_test_empty_decoder_config(0) - config.type = "sliding_window" - config.block_size = 6 - config.syndrome_size = 4 + # Inner decoder config + sw_config.inner_decoder_name = "multi_error_lut" + sw_config.multi_error_lut_params = qec.multi_error_lut_config() + sw_config.multi_error_lut_params.lut_error_depth = 2 - H = np.zeros((config.syndrome_size, config.block_size), dtype=np.uint8) - config.H_sparse = qec.pcm_to_sparse_vec(H) - O = np.zeros((1, config.block_size), dtype=np.uint8) - config.O_sparse = qec.pcm_to_sparse_vec(O) - config.D_sparse = qec.generate_timelike_sparse_detector_matrix( - config.syndrome_size, 2, include_first_round=False) - - config.decoder_custom_args = { - "window_size": 1, - "step_size": 1, - "num_syndromes_per_round": 2, - "num_boundary_syndromes": 1, - "error_rate_vec": [0.1] * config.block_size, - "inner_decoder_name": "single_error_lut", - } + config.set_decoder_custom_args(sw_config) multi_config.decoders = [config] check_decoder_yaml_roundtrip(multi_config) + check_decoder_creation(multi_config) if __name__ == "__main__": diff --git a/libs/qec/python/tests/test_decoding_config.py b/libs/qec/python/tests/test_decoding_config.py index 98c0255c2..a379246bc 100644 --- a/libs/qec/python/tests/test_decoding_config.py +++ b/libs/qec/python/tests/test_decoding_config.py @@ -6,15 +6,13 @@ # the terms of the Apache License 2.0 which accompanies this distribution. # # ============================================================================ # -import math - -import numpy as np import pytest +import numpy as np +import math import cudaq_qec as qec -# Decoder custom args are plain dicts. Their keys are governed by the -# parameter schema each decoder registers (see qec.decoder_param_schema). +# nv_qldpc_decoder_config tests def is_nv_qldpc_decoder_available(): @@ -25,361 +23,434 @@ def is_nv_qldpc_decoder_available(): H_list = [[1, 0, 0, 1, 0, 1, 1], [0, 1, 0, 1, 1, 0, 1], [0, 0, 1, 0, 1, 1, 1]] H_np = np.array(H_list, dtype=np.uint8) - qec.get_decoder("nv-qldpc-decoder", H_np) + nv_dec_gpu_and_cpu = qec.get_decoder("nv-qldpc-decoder", H_np) return True - except Exception: + except Exception as e: return False -# Schema introspection tests +FIELDS = { + "use_sparsity": (bool, True, False), + "error_rate": (float, 1e-3, 5e-2), + "error_rate_vec": (list, [0.01, 0.02, 0.03], [0.2, 0.1]), + "max_iterations": (int, 25, 50), + "n_threads": (int, 4, 8), + "use_osd": (bool, False, True), + "osd_method": (int, 1, 2), + "osd_order": (int, 7, 3), + "bp_batch_size": (int, 64, 128), + "osd_batch_size": (int, 16, 32), + "iter_per_check": (int, 2, 3), + "clip_value": (float, 10.0, 7.5), + "bp_method": (int, 0, 1), + "scale_factor": (float, 0.5, 1.25), + "proc_float": (str, "fp32", "fp64"), +} + + +def test_nv_qldpc_decoder_config_defaults_are_none(): + nv = qec.nv_qldpc_decoder_config() + for name in FIELDS: + assert getattr(nv, name) is None, f"Expected {name} to default to None" + + +@pytest.mark.parametrize("name, meta", list(FIELDS.items())) +def test_nv_qldpc_decoder_config_set_and_get_each_optional(name, meta): + nv = qec.nv_qldpc_decoder_config() + + py_type, sample_val, alt_val = meta + + # Initially None + assert getattr(nv, name) is None + + # Set to a valid value and get back + setattr(nv, name, sample_val) + got = getattr(nv, name) + if py_type is float: + assert isinstance(got, float) + assert math.isclose(got, float(sample_val), rel_tol=1e-12, abs_tol=0.0) + elif py_type is list: + assert isinstance(got, list) + assert all(isinstance(x, float) + for x in got), f"{name} must be a list of float" + assert got == sample_val + else: + assert isinstance(got, py_type) + assert got == sample_val + + # Change to an alternate valid value + setattr(nv, name, alt_val) + got2 = getattr(nv, name) + if py_type is float: + assert math.isclose(got2, float(alt_val), rel_tol=1e-12, abs_tol=0.0) + else: + assert got2 == alt_val + + # Set value to None + setattr(nv, name, None) + assert getattr(nv, name) is None + + +def test_nv_qldpc_decoder_config_setting_wrong_types_raises_typeerror(): + nv = qec.nv_qldpc_decoder_config() + + with pytest.raises(TypeError): + nv.max_iterations = "ten" + + with pytest.raises(TypeError): + nv.use_sparsity = "True" + + with pytest.raises(TypeError): + nv.error_rate = "0.1" + + with pytest.raises(TypeError): + nv.error_rate_vec = [0.1, "nope", 0.3] + + with pytest.raises(TypeError): + nv.error_rate_vec = 3.14 + + +def test_nv_qldpc_decoder_config_error_rate_vec_accepts_python_list_of_float(): + nv = qec.nv_qldpc_decoder_config() + + vals = [0.0, 0.125, 0.25] + nv.error_rate_vec = vals + got = nv.error_rate_vec + assert isinstance(got, list) + assert all(isinstance(x, float) for x in got) + assert got == vals + + +def test_nv_qldpc_decoder_config_toggle_multiple_fields_and_clear(): + nv = qec.nv_qldpc_decoder_config() + + nv.use_sparsity = True + nv.error_rate = 0.0123 + nv.error_rate_vec = [0.1, 0.2, 0.3] + nv.max_iterations = 100 + nv.n_threads = 8 + nv.use_osd = True + nv.osd_method = 2 + nv.osd_order = 4 + nv.bp_batch_size = 32 + nv.osd_batch_size = 16 + nv.iter_per_check = 3 + nv.clip_value = 7.5 + nv.bp_method = 1 + nv.scale_factor = 0.8 + nv.proc_float = "fp64" + assert nv is not None + assert nv.use_sparsity is True + assert math.isclose(nv.error_rate, 0.0123) + assert nv.error_rate_vec == [0.1, 0.2, 0.3] + assert nv.max_iterations == 100 + assert nv.n_threads == 8 -def test_decoder_param_schema_introspection(): - schema = qec.decoder_param_schema("nv-qldpc-decoder") - assert schema is not None - by_key = {entry["key"]: entry for entry in schema} - assert by_key["max_iterations"]["kind"] == "int32" - assert by_key["error_rate_vec"]["kind"] == "float64_vec" - assert by_key["srelay_config"]["kind"] == "subschema" - assert by_key["srelay_config"]["subschema"] == "srelay_bp" + nv.use_sparsity = None + nv.error_rate = None + nv.error_rate_vec = None + nv.max_iterations = None + nv.n_threads = None - sw_schema = qec.decoder_param_schema("sliding_window") - assert sw_schema is not None - by_key = {entry["key"]: entry for entry in sw_schema} - assert by_key["error_rate_vec"]["required"] is True - assert by_key["inner_decoder_params"]["kind"] == "discriminated" - assert by_key["inner_decoder_params"]["discriminator"] == \ - "inner_decoder_name" + assert nv.use_sparsity is None + assert nv.error_rate is None + assert nv.error_rate_vec is None + assert nv.max_iterations is None + assert nv.n_threads is None - assert qec.decoder_param_schema("no-such-decoder") is None - names = qec.registered_decoder_schemas() - assert "pymatching" in names - assert "multi_error_lut" in names +# multi_error_lut_config tests +FIELDS_MULTI_ERROR_LUT = { + "lut_error_depth": (int, 1, 3), +} -# decoder_config custom args tests +# pymatching_config tests +FIELDS_PYMATCHING = { + "error_rate_vec": (list, [0.1, 0.2, 0.3], [0.2, 0.1, 0.2]), + "merge_strategy": (str, "smallest_weight", "disallow"), +} -def test_decoder_custom_args_is_a_dict(): - dc = qec.decoder_config() - assert dc.decoder_custom_args == {} +FIELDS_CHROMOBIUS = { + "drop_mobius_errors_involving_remnant_errors": (bool, True, False), + "ignore_decomposition_failures": (bool, True, False), + "include_coords_in_mobius_dem": (bool, True, False), + "return_weight": (bool, True, False), + "write_mobius_match_to_stderr": (bool, True, False), +} - dc.decoder_custom_args = {"lut_error_depth": 2} - assert dc.decoder_custom_args == {"lut_error_depth": 2} +# trt_decoder_config tests - dc.set_decoder_custom_args({"lut_error_depth": 3}) - assert dc.decoder_custom_args == {"lut_error_depth": 3} +FIELDS_TRT_DECODER = { + "onnx_load_path": (str, "/path/to/model.onnx", "/other/path/model.onnx"), + "engine_load_path": (str, "/path/to/engine.trt", "/other/engine.trt"), + "engine_save_path": (str, "/path/to/save.trt", "/other/save.trt"), + "precision": (str, "fp16", "fp32"), + "memory_workspace": (int, 1073741824, 2147483648), # 1GB, 2GB +} -def test_decoder_config_yaml_roundtrip_and_custom_args(): - dc = qec.decoder_config() - dc.id = 0 - dc.type = "nv-qldpc-decoder" - dc.block_size = 10 - dc.syndrome_size = 3 - dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - dc.decoder_custom_args = { - "use_sparsity": True, - "error_rate": 0.01, - "max_iterations": 50, - "error_rate_vec": [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1], - "srelay_config": { - "pre_iter": 5, - "stopping_criterion": "NConv", - }, - } +def test_multi_error_lut_config_defaults_are_none(): + m = qec.multi_error_lut_config() + for name in FIELDS_MULTI_ERROR_LUT: + assert getattr(m, name) is None, f"Expected {name} to default to None" - yaml_text = dc.to_yaml_str() - assert isinstance(yaml_text, str) and len(yaml_text) > 0 - dc2 = qec.decoder_config.from_yaml_str(yaml_text) +def test_pymatching_config_defaults_are_none(): + pm = qec.pymatching_config() + for name in FIELDS_PYMATCHING: + assert getattr(pm, name) is None, f"Expected {name} to default to None" - # Basic scalar fields - assert dc2 is not None - assert dc2.id == 0 - assert dc2.type == "nv-qldpc-decoder" - assert dc2.block_size == 10 - assert dc2.syndrome_size == 3 - args = dc2.decoder_custom_args - assert args["use_sparsity"] is True - assert math.isclose(args["error_rate"], 0.01) - assert args["max_iterations"] == 50 - assert args["srelay_config"]["pre_iter"] == 5 - assert args["srelay_config"]["stopping_criterion"] == "NConv" +def test_chromobius_config_defaults_are_none(): + chromobius = qec.chromobius_config() + for name in FIELDS_CHROMOBIUS: + assert getattr(chromobius, name) is None -def test_pymatching_config_yaml_roundtrip(): - dc = qec.decoder_config() - dc.id = 0 - dc.type = "pymatching" - dc.block_size = 3 - dc.syndrome_size = 3 - dc.H_sparse = [0, -1, 1, -1, 2, -1] - dc.O_sparse = [0, -1, 1, -1, 2, -1] - dc.D_sparse = [0, -1, 1, -1, 2, -1] - dc.decoder_custom_args = { - "error_rate_vec": [0.1, 0.2, 0.3], - "merge_strategy": "smallest_weight", - } +@pytest.mark.parametrize("name, meta", list(FIELDS_PYMATCHING.items())) +def test_pymatching_config_set_and_get_each_optional(name, meta): + pm = qec.pymatching_config() - yaml_text = dc.to_yaml_str() - assert isinstance(yaml_text, str) and "pymatching" in yaml_text + py_type, sample_val, alt_val = meta - dc2 = qec.decoder_config.from_yaml_str(yaml_text) - assert dc2 is not None - assert dc2.type == "pymatching" + assert getattr(pm, name) is None - args = dc2.decoder_custom_args - assert list(args["error_rate_vec"]) == [0.1, 0.2, 0.3] - assert args["merge_strategy"] == "smallest_weight" + setattr(pm, name, sample_val) + got = getattr(pm, name) + assert isinstance(got, py_type) + assert got == sample_val + setattr(pm, name, alt_val) + got2 = getattr(pm, name) + assert got2 == alt_val -def test_unknown_custom_arg_key_is_rejected(): - dc = qec.decoder_config() - dc.id = 0 - dc.type = "pymatching" - dc.block_size = 3 - dc.syndrome_size = 3 - dc.H_sparse = [0, -1, 1, -1, 2, -1] - dc.O_sparse = [0, -1, 1, -1, 2, -1] - dc.D_sparse = [0, -1, 1, -1, 2, -1] - dc.decoder_custom_args = {"merge_strategy": "smallest_weight"} - yaml_text = qec_yaml_for(dc) + setattr(pm, name, None) + assert getattr(pm, name) is None - misspelled = yaml_text.replace("merge_strategy", "merge_stratgey") - with pytest.raises(RuntimeError): - qec.multi_decoder_config.from_yaml_str(misspelled) +@pytest.mark.parametrize("name, meta", list(FIELDS_CHROMOBIUS.items())) +def test_chromobius_config_set_and_get_each_optional(name, meta): + chromobius = qec.chromobius_config() -def test_validate_custom_args_checks_dict_built_configs(): - # Dicts assigned to decoder_custom_args never pass through the YAML - # parser; validate_custom_args applies the same schema checks explicitly. - dc = qec.decoder_config() - dc.type = "pymatching" - dc.decoder_custom_args = {"merge_strategy": "smallest_weight"} - dc.validate_custom_args() + py_type, sample_val, alt_val = meta - dc.decoder_custom_args = {"merge_stratgey": "smallest_weight"} - with pytest.raises(RuntimeError, match="merge_stratgey"): - dc.validate_custom_args() + assert getattr(chromobius, name) is None - # Non-empty args for a type with no registered schema are rejected. - dc.type = "decoder_without_registered_schema" - dc.decoder_custom_args = {"anything": 1} - with pytest.raises(RuntimeError, match="no registered parameter schema"): - dc.validate_custom_args() + setattr(chromobius, name, sample_val) + got = getattr(chromobius, name) + assert isinstance(got, py_type) + assert got == sample_val - # multi_decoder_config validates every decoder. - mdc = qec.multi_decoder_config() - mdc.decoders = [dc] - with pytest.raises(RuntimeError): - mdc.validate_custom_args() + setattr(chromobius, name, alt_val) + got2 = getattr(chromobius, name) + assert got2 == alt_val + + setattr(chromobius, name, None) + assert getattr(chromobius, name) is None -def test_custom_args_dict_values_convert_to_schema_types(): - # The setter converts dict values to the canonical types the registered - # schema declares: Python ints are accepted for f64 params and negative - # ints for int32 params (the generic conversion stores every int as - # size_t, which would reject both). +def test_configure_valid_multi_error_lut_decoders(): + nv = qec.multi_error_lut_config() + nv.lut_error_depth = 2 + dc = qec.decoder_config() dc.id = 0 - dc.type = "nv-qldpc-decoder" - dc.block_size = 1 - dc.syndrome_size = 1 - dc.H_sparse = [0, -1] - dc.O_sparse = [0, -1] - dc.D_sparse = [0, -1] + dc.type = "multi_error_lut" + dc.block_size = 10 + dc.syndrome_size = 3 + dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] + dc.D_sparse = qec.generate_timelike_sparse_detector_matrix( + dc.syndrome_size, 2, include_first_round=False) + dc.set_decoder_custom_args(nv) - dc.decoder_custom_args = {"clip_value": 2} # int for an f64 param - assert "clip_value" in dc.to_yaml_str() + mdc = qec.multi_decoder_config() + mdc.decoders = [dc] + ret = qec.configure_decoders(mdc) + qec.finalize_decoders() + assert isinstance(ret, int) + assert ret == 0 - dc.decoder_custom_args = {"bp_seed": -1} # negative int32 - assert dc.decoder_custom_args == {"bp_seed": -1} - round_tripped = qec.multi_decoder_config.from_yaml_str(qec_yaml_for(dc)) - assert round_tripped.decoders[0].decoder_custom_args == {"bp_seed": -1} - # Mismatched types raise a clear error naming the parameter. - with pytest.raises(RuntimeError, match="bp_seed"): - dc.decoder_custom_args = {"bp_seed": "oops"} +# trt_decoder_config tests -def test_decoder_config_from_yaml_str_rejects_malformed_yaml(): - with pytest.raises(RuntimeError, match="Invalid decoder configuration"): - qec.decoder_config.from_yaml_str("id: [oops") +def test_trt_decoder_config_defaults_are_none(): + trt = qec.trt_decoder_config() + for name in FIELDS_TRT_DECODER: + assert getattr(trt, name) is None, f"Expected {name} to default to None" -def test_validate_custom_args_runs_schema_validate_hook(): - # sliding_window registers a validate hook for cross-field constraints - # (step_size must be between 1 and window_size). - dc = qec.decoder_config() - dc.type = "sliding_window" - dc.decoder_custom_args = { - "window_size": 4, - "step_size": 2, - "error_rate_vec": [0.01, 0.01], - "inner_decoder_name": "single_error_lut", - } - dc.validate_custom_args() - - dc.decoder_custom_args = { - "window_size": 2, - "step_size": 4, - "error_rate_vec": [0.01, 0.01], - "inner_decoder_name": "single_error_lut", - } - with pytest.raises(RuntimeError, match="step_size"): - dc.validate_custom_args() - - # Missing required key (error_rate_vec). - dc.decoder_custom_args = {"inner_decoder_name": "single_error_lut"} - with pytest.raises(RuntimeError, match="error_rate_vec"): - dc.validate_custom_args() - - -def test_decoder_config_json_schema_validates_yaml_documents(): - # The exported JSON Schema lets standard third-party tooling validate - # user-provided configuration YAML without loading this library. - jsonschema = pytest.importorskip("jsonschema") - yaml = pytest.importorskip("yaml") - import json - - schema = json.loads(qec.decoder_config_json_schema()) - jsonschema.Draft202012Validator.check_schema(schema) - validator = jsonschema.Draft202012Validator(schema) - - # Every registered decoder schema appears in the export. - assert set(qec.registered_decoder_schemas()) == set( - schema["$defs"]["decoder_params"]) - - # A real emitted configuration validates. - dc = qec.decoder_config() - dc.id = 0 - dc.type = "pymatching" - dc.block_size = 3 - dc.syndrome_size = 3 - dc.H_sparse = [0, -1, 1, -1, 2, -1] - dc.O_sparse = [0, -1, 1, -1, 2, -1] - dc.D_sparse = [0, -1, 1, -1, 2, -1] - dc.decoder_custom_args = { - "error_rate_vec": [0.1, 0.1, 0.1], - "merge_strategy": "smallest_weight", - } - document = yaml.safe_load(qec_yaml_for(dc)) - validator.validate(document) - - # Unknown custom-arg keys fail validation. - bad = yaml.safe_load(qec_yaml_for(dc)) - bad["decoders"][0]["decoder_custom_args"]["merge_stratgey"] = \ - bad["decoders"][0]["decoder_custom_args"].pop("merge_strategy") - with pytest.raises(jsonschema.ValidationError): - validator.validate(bad) - - # Missing required custom-arg keys fail validation (sliding_window - # requires error_rate_vec and inner_decoder_name). - missing = yaml.safe_load(qec_yaml_for(dc)) - missing["decoders"][0]["type"] = "sliding_window" - missing["decoders"][0]["decoder_custom_args"] = {"window_size": 2} - with pytest.raises(jsonschema.ValidationError): - validator.validate(missing) - - # Custom args for a type with no registered schema fail validation. - unregistered = yaml.safe_load(qec_yaml_for(dc)) - unregistered["decoders"][0]["type"] = "decoder_without_registered_schema" - with pytest.raises(jsonschema.ValidationError): - validator.validate(unregistered) - - # Missing envelope fields fail validation. - no_type = yaml.safe_load(qec_yaml_for(dc)) - del no_type["decoders"][0]["type"] - with pytest.raises(jsonschema.ValidationError): - validator.validate(no_type) - - -def qec_yaml_for(dc): - mdc = qec.multi_decoder_config() - mdc.decoders = [dc] - return mdc.to_yaml_str() +@pytest.mark.parametrize("name, meta", list(FIELDS_TRT_DECODER.items())) +def test_trt_decoder_config_set_and_get_each_optional(name, meta): + trt = qec.trt_decoder_config() + py_type, sample_val, alt_val = meta -# trt_decoder tests (need the trt_decoder plugin for its parameter schema) + # Initially None + assert getattr(trt, name) is None -trt_schema_missing = qec.decoder_param_schema("trt_decoder") is None + # Set to a valid value and get back + setattr(trt, name, sample_val) + got = getattr(trt, name) + assert isinstance(got, py_type) + assert got == sample_val + + # Change to an alternate valid value + setattr(trt, name, alt_val) + got2 = getattr(trt, name) + assert got2 == alt_val + + # Set value to None + setattr(trt, name, None) + assert getattr(trt, name) is None -@pytest.mark.skipif( - trt_schema_missing, - reason="trt_decoder plugin (and its parameter schema) not available") def test_trt_decoder_config_yaml_roundtrip(): + trt = qec.trt_decoder_config() + trt.engine_load_path = "/path/to/engine.trt" + trt.precision = "fp16" + trt.memory_workspace = 1073741824 # 1GB + dc = qec.decoder_config() dc.id = 0 dc.type = "trt_decoder" dc.block_size = 10 dc.syndrome_size = 3 dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - dc.decoder_custom_args = { - "engine_load_path": "/path/to/engine.trt", - "precision": "fp16", - "memory_workspace": 1073741824, # 1GB - } + dc.set_decoder_custom_args(trt) yaml_text = dc.to_yaml_str() assert isinstance(yaml_text, str) and len(yaml_text) > 0 dc2 = qec.decoder_config.from_yaml_str(yaml_text) + # Basic scalar fields assert dc2 is not None assert dc2.id == 0 assert dc2.type == "trt_decoder" + assert dc2.block_size == 10 + assert dc2.syndrome_size == 3 - args = dc2.decoder_custom_args - assert args["engine_load_path"] == "/path/to/engine.trt" - assert args["precision"] == "fp16" - assert args["memory_workspace"] == 1073741824 + # Recover TRT config from decoder_custom_args + trt2 = dc2.decoder_custom_args + assert trt2 is not None + assert trt2.engine_load_path == "/path/to/engine.trt" + assert trt2.precision == "fp16" + assert trt2.memory_workspace == 1073741824 -@pytest.mark.skipif( - trt_schema_missing, - reason="trt_decoder plugin (and its parameter schema) not available") -def test_trt_decoder_chromobius_global_config_yaml_roundtrip(): +def test_trt_decoder_config_chromobius_global_params_roundtrip(): + trt = qec.trt_decoder_config() + chromobius = qec.chromobius_config() + chromobius.return_weight = True + + trt.global_decoder = "chromobius" + trt.global_decoder_params = chromobius + + got = trt.global_decoder_params + assert isinstance(got, qec.chromobius_config) + assert got.return_weight is True + + as_map = trt.to_heterogeneous_map() + assert as_map["global_decoder"] == "chromobius" + assert as_map["global_decoder_params"]["return_weight"] is True + + trt2 = qec.trt_decoder_config.from_heterogeneous_map(as_map) + got2 = trt2.global_decoder_params + assert isinstance(got2, qec.chromobius_config) + assert got2.return_weight is True + + trt2.global_decoder_params = None + assert trt2.global_decoder_params is None + + +def test_trt_decoder_config_defaults_omitted_global_params(): + for global_decoder, config_type in ( + ("pymatching", qec.pymatching_config), + ("chromobius", qec.chromobius_config), + ): + trt = qec.trt_decoder_config.from_heterogeneous_map( + {"global_decoder": global_decoder}) + + got = trt.global_decoder_params + assert isinstance(got, config_type) + + as_map = trt.to_heterogeneous_map() + assert as_map["global_decoder"] == global_decoder + assert as_map["global_decoder_params"] == {} + + trt = qec.trt_decoder_config() + trt.global_decoder = global_decoder + as_map = trt.to_heterogeneous_map() + assert as_map["global_decoder"] == global_decoder + assert as_map["global_decoder_params"] == {} + + +def test_trt_decoder_config_preserves_unknown_omitted_global_params(): + trt = qec.trt_decoder_config.from_heterogeneous_map( + {"global_decoder": "my_plugin"}) + + assert trt.global_decoder_params is None + + as_map = trt.to_heterogeneous_map() + assert as_map["global_decoder"] == "my_plugin" + assert "global_decoder_params" not in as_map + + trt = qec.trt_decoder_config() + trt.global_decoder = "my_plugin" + as_map = trt.to_heterogeneous_map() + assert as_map["global_decoder"] == "my_plugin" + assert "global_decoder_params" not in as_map + + +def test_trt_decoder_config_rejects_unknown_global_params(): + with pytest.raises(RuntimeError): + qec.trt_decoder_config.from_heterogeneous_map({ + "global_decoder": "my_plugin", + "global_decoder_params": {}, + }) + + +def test_pymatching_config_yaml_roundtrip(): + pm = qec.pymatching_config() + pm.error_rate_vec = [0.1, 0.2, 0.3] + pm.merge_strategy = "smallest_weight" + dc = qec.decoder_config() dc.id = 0 - dc.type = "trt_decoder" + dc.type = "pymatching" dc.block_size = 3 dc.syndrome_size = 3 dc.H_sparse = [0, -1, 1, -1, 2, -1] dc.O_sparse = [0, -1, 1, -1, 2, -1] dc.D_sparse = [0, -1, 1, -1, 2, -1] - dc.decoder_custom_args = { - "onnx_load_path": "/tmp/predecoder.onnx", - "global_decoder": "chromobius", - "global_decoder_params": { - "ignore_decomposition_failures": True, - "return_weight": False, - }, - } + dc.set_decoder_custom_args(pm) yaml_text = dc.to_yaml_str() - assert isinstance(yaml_text, str) and "chromobius" in yaml_text + assert isinstance(yaml_text, str) and "pymatching" in yaml_text dc2 = qec.decoder_config.from_yaml_str(yaml_text) assert dc2 is not None - assert dc2.type == "trt_decoder" + assert dc2.type == "pymatching" + + pm2 = dc2.decoder_custom_args + assert pm2 is not None + assert pm2.error_rate_vec == [0.1, 0.2, 0.3] + assert pm2.merge_strategy == "smallest_weight" + - args = dc2.decoder_custom_args - assert args["global_decoder"] == "chromobius" - assert args["global_decoder_params"]["ignore_decomposition_failures"] \ - is True - assert args["global_decoder_params"]["return_weight"] is False +def test_trt_decoder_chromobius_global_config_yaml_roundtrip(): + chromobius = qec.chromobius_config() + chromobius.ignore_decomposition_failures = True + chromobius.return_weight = False + trt = qec.trt_decoder_config() + trt.global_decoder = "chromobius" + trt.global_decoder_params = chromobius -@pytest.mark.skipif( - trt_schema_missing, - reason="trt_decoder plugin (and its parameter schema) not available") -def test_trt_decoder_default_global_params_materialized(): - # A named global decoder with a registered schema gets an empty params - # section materialized on parse. dc = qec.decoder_config() dc.id = 0 dc.type = "trt_decoder" @@ -388,105 +459,85 @@ def test_trt_decoder_default_global_params_materialized(): dc.H_sparse = [0, -1, 1, -1, 2, -1] dc.O_sparse = [0, -1, 1, -1, 2, -1] dc.D_sparse = [0, -1, 1, -1, 2, -1] - dc.decoder_custom_args = {"global_decoder": "pymatching"} + dc.set_decoder_custom_args(trt) + + yaml_text = dc.to_yaml_str() + assert isinstance(yaml_text, str) and "chromobius" in yaml_text - mdc2 = qec.multi_decoder_config.from_yaml_str(qec_yaml_for(dc)) - args = mdc2.decoders[0].decoder_custom_args - assert args["global_decoder_params"] == {} + dc2 = qec.decoder_config.from_yaml_str(yaml_text) + assert dc2 is not None + assert dc2.type == "trt_decoder" - # ... and already on FIRST emission (matching the old typed path), so - # emitted YAML is stable across round trips. - first = qec_yaml_for(dc) - assert "global_decoder_params" in first - assert mdc2.to_yaml_str() == first + trt2 = dc2.decoder_custom_args + assert trt2 is not None + assert trt2.global_decoder == "chromobius" + chromobius2 = trt2.global_decoder_params + assert chromobius2 is not None + assert chromobius2.ignore_decomposition_failures is True + assert chromobius2.return_weight is False -def test_validate_custom_args_checks_value_kinds(): - # A dict assigned before `type` is set takes the generic conversion - # (ints stored as size_t), which an f64 param cannot read back at - # emission. validate_custom_args must name the offending key instead of - # letting emission fail with a low-context error later. - dc = qec.decoder_config() - dc.decoder_custom_args = {"clip_value": 2} # type not set yet - dc.type = "nv-qldpc-decoder" - with pytest.raises(RuntimeError, match="clip_value"): - dc.validate_custom_args() - # Assigned after `type`, the same dict converts to the schema's declared - # types and validates. - dc.decoder_custom_args = {"clip_value": 2} - dc.validate_custom_args() +# decoder_config tests -def test_non_schema_keys_dropped_from_emission_and_decoder_params(): - # A key outside the registered schema cannot round-trip through YAML, so - # it is warned-and-dropped from the emitted YAML (and from the map local - # decoders receive) rather than taking effect locally but silently - # vanishing when the config is serialized for a remote target. +def test_decoder_config_yaml_roundtrip_and_custom_args(): + # Build NV config and embed into DecoderConfig via helper + nv = qec.nv_qldpc_decoder_config() + nv.use_sparsity = True + nv.error_rate = 0.01 + nv.max_iterations = 50 + nv.error_rate_vec = [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1] + dc = qec.decoder_config() dc.id = 0 - dc.type = "multi_error_lut" - dc.block_size = 3 + dc.type = "nv-qldpc-decoder" + dc.block_size = 10 dc.syndrome_size = 3 - dc.H_sparse = [0, -1, 1, -1, 2, -1] - dc.decoder_custom_args = {"lut_error_depth": 2, "not_a_real_param": 42} + dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] + dc.set_decoder_custom_args(nv) - yaml_text = qec_yaml_for(dc) - assert "lut_error_depth" in yaml_text - assert "not_a_real_param" not in yaml_text + yaml_text = dc.to_yaml_str() + assert isinstance(yaml_text, str) and len(yaml_text) > 0 - # The stored args are untouched; validate_custom_args still rejects them - # for callers who want a hard error instead of the warn-and-drop. - assert dc.decoder_custom_args["not_a_real_param"] == 42 - with pytest.raises(RuntimeError, match="not_a_real_param"): - dc.validate_custom_args() + dc2 = qec.decoder_config.from_yaml_str(yaml_text) + # Basic scalar fields + assert dc2 is not None + assert dc2.id == 0 + assert dc2.type == "nv-qldpc-decoder" + assert dc2.block_size == 10 + assert dc2.syndrome_size == 3 -@pytest.mark.skipif( - trt_schema_missing, - reason="trt_decoder plugin (and its parameter schema) not available") -def test_trt_decoder_rejects_unknown_global_params(): - dc = qec.decoder_config() - dc.id = 0 - dc.type = "trt_decoder" - dc.block_size = 3 - dc.syndrome_size = 3 - dc.H_sparse = [0, -1, 1, -1, 2, -1] - dc.O_sparse = [0, -1, 1, -1, 2, -1] - dc.D_sparse = [0, -1, 1, -1, 2, -1] - yaml_text = """ -decoders: - - id: 0 - type: trt_decoder - block_size: 3 - syndrome_size: 3 - H_sparse: [0, -1, 1, -1, 2, -1] - O_sparse: [0, -1, 1, -1, 2, -1] - D_sparse: [0, -1, 1, -1, 2, -1] - decoder_custom_args: - global_decoder: my_plugin - global_decoder_params: {} -""" - with pytest.raises(RuntimeError): - qec.multi_decoder_config.from_yaml_str(yaml_text) + # Recover NV config from decoder_custom_args (it's already the config object) + nv2 = dc2.decoder_custom_args + assert nv2 is not None + assert nv2.use_sparsity is True + assert math.isclose(nv2.error_rate, 0.01) + assert nv2.max_iterations == 50 # multi_decoder_config tests def test_multi_decoder_config_yaml_roundtrip(): + # Build NV config and embed into DecoderConfig via helper + nv = qec.nv_qldpc_decoder_config() + nv.use_sparsity = True + nv.error_rate = 0.01 + nv.error_rate_vec = [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1] + nv.max_iterations = 50 + d1 = qec.decoder_config() d1.id = 0 d1.type = "nv-qldpc-decoder" d1.block_size = 10 d1.syndrome_size = 3 d1.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - d1.decoder_custom_args = { - "use_sparsity": True, - "error_rate": 0.01, - "error_rate_vec": [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1], - "max_iterations": 50, - } + d1.set_decoder_custom_args(nv) + + lut_config = qec.multi_error_lut_config() + lut_config.lut_error_depth = 3 d2 = qec.decoder_config() d2.id = 1 @@ -494,7 +545,7 @@ def test_multi_decoder_config_yaml_roundtrip(): d2.block_size = 10 d2.syndrome_size = 3 d2.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - d2.decoder_custom_args = {"lut_error_depth": 3} + d2.set_decoder_custom_args(lut_config) mdc = qec.multi_decoder_config() mdc.decoders = [d1, d2] @@ -507,29 +558,6 @@ def test_multi_decoder_config_yaml_roundtrip(): assert len(mdc2.decoders) == 2 ids = sorted({md.id for md in mdc2.decoders}) assert ids == [0, 1] - assert mdc2 == mdc - - -# configure_decoders tests - - -def test_configure_valid_multi_error_lut_decoders(): - dc = qec.decoder_config() - dc.id = 0 - dc.type = "multi_error_lut" - dc.block_size = 10 - dc.syndrome_size = 3 - dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - dc.D_sparse = qec.generate_timelike_sparse_detector_matrix( - dc.syndrome_size, 2, include_first_round=False) - dc.decoder_custom_args = {"lut_error_depth": 2} - - mdc = qec.multi_decoder_config() - mdc.decoders = [dc] - ret = qec.configure_decoders(mdc) - qec.finalize_decoders() - assert isinstance(ret, int) - assert ret == 0 def test_configure_decoders_from_str_smoke(): @@ -539,15 +567,16 @@ def test_configure_decoders_from_str_smoke(): assert isinstance(status, int) qec.finalize_decoders() + nv = qec.nv_qldpc_decoder_config() + nv.error_rate_vec = [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1] + decoder_config = qec.decoder_config() decoder_config.id = 0 decoder_config.type = "nv-qldpc-decoder" decoder_config.block_size = 10 decoder_config.syndrome_size = 3 decoder_config.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - decoder_config.decoder_custom_args = { - "error_rate_vec": [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1], - } + decoder_config.set_decoder_custom_args(nv) multi_decoder_config = qec.multi_decoder_config() multi_decoder_config.decoders = [decoder_config] yaml_str = multi_decoder_config.to_yaml_str() @@ -568,7 +597,34 @@ def test_configure_decoders_from_file_smoke(tmp_path): qec.finalize_decoders() -def make_pymatching_multi_decoder_config(pm_args, h_sparse=None): +def test_configure_valid_decoders(): + nv = qec.nv_qldpc_decoder_config() + nv.use_sparsity = True + nv.error_rate = 0.01 + nv.error_rate_vec = [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1] + nv.max_iterations = 50 + + dc = qec.decoder_config() + dc.id = 0 + dc.type = "multi_error_lut" + dc.block_size = 10 + dc.syndrome_size = 3 + dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] + dc.D_sparse = qec.generate_timelike_sparse_detector_matrix( + dc.syndrome_size, 2, include_first_round=False) + lut_config = qec.multi_error_lut_config() + lut_config.lut_error_depth = 2 + dc.set_decoder_custom_args(lut_config) + + mdc = qec.multi_decoder_config() + mdc.decoders = [dc] + ret = qec.configure_decoders(mdc) + qec.finalize_decoders() + assert isinstance(ret, int) + assert ret == 0 + + +def make_pymatching_multi_decoder_config(pm, h_sparse=None): dc = qec.decoder_config() dc.id = 0 dc.type = "pymatching" @@ -577,26 +633,27 @@ def make_pymatching_multi_decoder_config(pm_args, h_sparse=None): dc.H_sparse = h_sparse if h_sparse is not None else [0, -1, 1, -1, 2, -1] dc.O_sparse = [0, -1, 1, -1, 2, -1] dc.D_sparse = [0, -1, 1, -1, 2, -1] - dc.decoder_custom_args = pm_args + dc.set_decoder_custom_args(pm) mdc = qec.multi_decoder_config() mdc.decoders = [dc] return mdc -def configure_pymatching_status(pm_args, h_sparse=None): +def configure_pymatching_status(pm, h_sparse=None): try: return qec.configure_decoders( - make_pymatching_multi_decoder_config(pm_args, h_sparse)) + make_pymatching_multi_decoder_config(pm, h_sparse)) finally: qec.finalize_decoders() def test_configure_valid_pymatching_decoder(): - ret = configure_pymatching_status({ - "error_rate_vec": [0.1, 0.1, 0.1], - "merge_strategy": "smallest_weight", - }) + pm = qec.pymatching_config() + pm.error_rate_vec = [0.1, 0.1, 0.1] + pm.merge_strategy = "smallest_weight" + + ret = configure_pymatching_status(pm) assert isinstance(ret, int) assert ret == 0 @@ -606,42 +663,49 @@ def test_configure_valid_pymatching_decoder(): ([0.1, 0.1], [0.0, 0.1, 0.1], [0.1, 0.6, 0.1]), ) def test_configure_invalid_pymatching_error_rate_vec(error_rate_vec): - ret = configure_pymatching_status({ - "error_rate_vec": error_rate_vec, - "merge_strategy": "smallest_weight", - }) + pm = qec.pymatching_config() + pm.error_rate_vec = error_rate_vec + pm.merge_strategy = "smallest_weight" + + ret = configure_pymatching_status(pm) assert isinstance(ret, int) assert ret != 0 def test_configure_invalid_pymatching_merge_strategy(): - ret = configure_pymatching_status({ - "error_rate_vec": [0.1, 0.1, 0.1], - "merge_strategy": "not-a-strategy", - }) + pm = qec.pymatching_config() + pm.error_rate_vec = [0.1, 0.1, 0.1] + pm.merge_strategy = "not-a-strategy" + + ret = configure_pymatching_status(pm) assert isinstance(ret, int) assert ret != 0 def test_configure_invalid_pymatching_non_graphlike_h_sparse(): - ret = configure_pymatching_status( - { - "error_rate_vec": [0.1, 0.1, 0.1], - "merge_strategy": "smallest_weight", - }, - h_sparse=[0, -1, 0, -1, 0, -1]) + pm = qec.pymatching_config() + pm.error_rate_vec = [0.1, 0.1, 0.1] + pm.merge_strategy = "smallest_weight" + + ret = configure_pymatching_status(pm, h_sparse=[0, -1, 0, -1, 0, -1]) assert isinstance(ret, int) assert ret != 0 def test_configure_invalid_decoders(): + nv = qec.nv_qldpc_decoder_config() + nv.use_sparsity = True + nv.error_rate = 0.01 + nv.error_rate_vec = [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1] + nv.max_iterations = 50 + decoder_config = qec.decoder_config() decoder_config.id = 0 decoder_config.type = "invalid-decoder" decoder_config.block_size = 10 decoder_config.syndrome_size = 3 decoder_config.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - decoder_config.decoder_custom_args = {"max_iterations": 50} + decoder_config.set_decoder_custom_args(nv) multi_decoder_config = qec.multi_decoder_config() multi_decoder_config.decoders = [decoder_config] diff --git a/libs/qec/python/tests/test_decoding_config_deprecated.py b/libs/qec/python/tests/test_decoding_config_deprecated.py deleted file mode 100644 index cede41ff4..000000000 --- a/libs/qec/python/tests/test_decoding_config_deprecated.py +++ /dev/null @@ -1,891 +0,0 @@ -# ============================================================================ # -# Copyright (c) 2024 - 2026 NVIDIA Corporation & Affiliates. # -# All rights reserved. # -# # -# This source code and the accompanying materials are made available under # -# the terms of the Apache License 2.0 which accompanies this distribution. # -# ============================================================================ # -"""Tests for the deprecated typed decoder-config classes. - -The typed config classes (nv_qldpc_decoder_config and friends) are deprecated -dict-backed shims over the schema-driven interface (see cudaq_qec/_compat.py). -This module proves they still behave like the originals: the bulk of it is the -pre-schema typed-config test suite restored verbatim, plus shim-specific tests -(deprecation warnings, dict equivalence). The only edits to the restored tests -are marked inline: reading decoder_config.decoder_custom_args now returns a -plain dict, never a typed object. Delete this file together with _compat.py -when the deprecation period ends. -""" - -import math - -import numpy as np -import pytest - -import cudaq_qec as qec - -# The deprecated classes warn on every construction; that is asserted once in -# test_deprecated_typed_configs_warn_on_construction and silenced everywhere -# else so the restored tests run unmodified. -pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning") - -trt_schema_missing = qec.decoder_param_schema("trt_decoder") is None -chromobius_schema_missing = qec.decoder_param_schema("chromobius") is None - - -def is_nv_qldpc_decoder_available(): - """ - Helper function to check if the NV-QLDPC decoder is available. - """ - try: - H_list = [[1, 0, 0, 1, 0, 1, 1], [0, 1, 0, 1, 1, 0, 1], - [0, 0, 1, 0, 1, 1, 1]] - H_np = np.array(H_list, dtype=np.uint8) - qec.get_decoder("nv-qldpc-decoder", H_np) - return True - except Exception: - return False - - -# Shim-specific tests (not part of the restored pre-schema suite) - - -def make_nv_qldpc_decoder_config(id=0): - dc = qec.decoder_config() - dc.id = id - dc.type = "nv-qldpc-decoder" - dc.block_size = 10 - dc.syndrome_size = 3 - dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - return dc - - -def test_deprecated_typed_configs_warn_on_construction(): - for cls in (qec.nv_qldpc_decoder_config, qec.multi_error_lut_config, - qec.trt_decoder_config, qec.pymatching_config, - qec.chromobius_config, qec.qecrt.config.srelay_bp_config, - qec.qecrt.config.single_error_lut_config, - qec.qecrt.config.sliding_window_config): - with pytest.warns(DeprecationWarning): - cls() - - -def test_deprecated_config_matches_dict_built_yaml(): - cfg = qec.nv_qldpc_decoder_config() - cfg.use_sparsity = True - cfg.error_rate = 0.01 - cfg.max_iterations = 50 - cfg.bp_seed = -1 - cfg.srelay_config = qec.qecrt.config.srelay_bp_config() - cfg.srelay_config.pre_iter = 5 - cfg.srelay_config.stopping_criterion = "NConv" - - old_style = make_nv_qldpc_decoder_config() - old_style.set_decoder_custom_args(cfg) - - new_style = make_nv_qldpc_decoder_config() - new_style.decoder_custom_args = { - "use_sparsity": True, - "error_rate": 0.01, - "max_iterations": 50, - "bp_seed": -1, - "srelay_config": { - "pre_iter": 5, - "stopping_criterion": "NConv", - }, - } - - assert old_style.to_yaml_str() == new_style.to_yaml_str() - assert old_style == new_style - - -def test_deprecated_config_assignable_to_property(): - cfg = qec.pymatching_config() - cfg.error_rate_vec = [0.1, 0.2, 0.3] - cfg.merge_strategy = "smallest_weight" - - dc = qec.decoder_config() - # The shim carries its own schema name, so conversion works even before - # dc.type is assigned. - dc.decoder_custom_args = cfg - dc.type = "pymatching" - args = dc.decoder_custom_args - assert list(args["error_rate_vec"]) == [0.1, 0.2, 0.3] - assert args["merge_strategy"] == "smallest_weight" - dc.validate_custom_args() - - -def test_deprecated_config_unset_fields_read_as_none_and_are_omitted(): - cfg = qec.nv_qldpc_decoder_config() - assert cfg.max_iterations is None - cfg.max_iterations = 50 - assert cfg.max_iterations == 50 - cfg.max_iterations = None # clears, like the old std::optional fields - assert cfg.to_heterogeneous_map() == {} - - -def test_deprecated_config_rejects_unknown_attributes(): - cfg = qec.pymatching_config() - with pytest.raises(AttributeError): - cfg.merge_stratgey = "smallest_weight" - with pytest.raises(AttributeError): - _ = cfg.merge_stratgey - - -def test_deprecated_sliding_window_inner_params_collapse(): - cfg = qec.qecrt.config.sliding_window_config() - cfg.window_size = 3 - cfg.step_size = 1 - cfg.error_rate_vec = [0.1, 0.2, 0.3] - cfg.inner_decoder_name = "multi_error_lut" - cfg.multi_error_lut_params = qec.multi_error_lut_config() - cfg.multi_error_lut_params.lut_error_depth = 2 - - assert cfg.to_heterogeneous_map() == { - "window_size": 3, - "step_size": 1, - "error_rate_vec": [0.1, 0.2, 0.3], - "inner_decoder_name": "multi_error_lut", - "inner_decoder_params": { - "lut_error_depth": 2 - }, - } - - dc = qec.decoder_config() - dc.id = 0 - dc.type = "sliding_window" - dc.block_size = 3 - dc.syndrome_size = 3 - dc.H_sparse = [0, -1, 1, -1, 2, -1] - dc.set_decoder_custom_args(cfg) - dc.validate_custom_args() - - round_tripped = qec.decoder_config.from_yaml_str(dc.to_yaml_str()) - args = round_tripped.decoder_custom_args - assert args["inner_decoder_params"]["lut_error_depth"] == 2 - - -def test_deprecated_config_from_heterogeneous_map_round_trip(): - source = { - "window_size": 3, - "error_rate_vec": [0.1, 0.2, 0.3], - "inner_decoder_name": "multi_error_lut", - "inner_decoder_params": { - "lut_error_depth": 2 - }, - } - cfg = qec.qecrt.config.sliding_window_config.from_heterogeneous_map(source) - assert cfg.window_size == 3 - assert cfg.inner_decoder_name == "multi_error_lut" - assert cfg.multi_error_lut_params.lut_error_depth == 2 - assert cfg.to_heterogeneous_map() == source - - -def test_deprecated_config_schema_validation_still_applies(): - cfg = qec.qecrt.config.sliding_window_config() - cfg.window_size = 2 - cfg.step_size = 5 # step_size > window_size: schema validate hook rejects - cfg.error_rate_vec = [0.1] - cfg.inner_decoder_name = "single_error_lut" - - dc = qec.decoder_config() - dc.type = "sliding_window" - dc.set_decoder_custom_args(cfg) - with pytest.raises(RuntimeError, match="step_size"): - dc.validate_custom_args() - - -# --------------------------------------------------------------------------- -# The pre-schema typed-config test suite, restored verbatim (except where -# marked) from the version this branch removed. -# --------------------------------------------------------------------------- - -# nv_qldpc_decoder_config tests - -FIELDS = { - "use_sparsity": (bool, True, False), - "error_rate": (float, 1e-3, 5e-2), - "error_rate_vec": (list, [0.01, 0.02, 0.03], [0.2, 0.1]), - "max_iterations": (int, 25, 50), - "n_threads": (int, 4, 8), - "use_osd": (bool, False, True), - "osd_method": (int, 1, 2), - "osd_order": (int, 7, 3), - "bp_batch_size": (int, 64, 128), - "osd_batch_size": (int, 16, 32), - "iter_per_check": (int, 2, 3), - "clip_value": (float, 10.0, 7.5), - "bp_method": (int, 0, 1), - "scale_factor": (float, 0.5, 1.25), - "proc_float": (str, "fp32", "fp64"), -} - - -def test_nv_qldpc_decoder_config_defaults_are_none(): - nv = qec.nv_qldpc_decoder_config() - for name in FIELDS: - assert getattr(nv, name) is None, f"Expected {name} to default to None" - - -@pytest.mark.parametrize("name, meta", list(FIELDS.items())) -def test_nv_qldpc_decoder_config_set_and_get_each_optional(name, meta): - nv = qec.nv_qldpc_decoder_config() - - py_type, sample_val, alt_val = meta - - # Initially None - assert getattr(nv, name) is None - - # Set to a valid value and get back - setattr(nv, name, sample_val) - got = getattr(nv, name) - if py_type is float: - assert isinstance(got, float) - assert math.isclose(got, float(sample_val), rel_tol=1e-12, abs_tol=0.0) - elif py_type is list: - assert isinstance(got, list) - assert all(isinstance(x, float) - for x in got), f"{name} must be a list of float" - assert got == sample_val - else: - assert isinstance(got, py_type) - assert got == sample_val - - # Change to an alternate valid value - setattr(nv, name, alt_val) - got2 = getattr(nv, name) - if py_type is float: - assert math.isclose(got2, float(alt_val), rel_tol=1e-12, abs_tol=0.0) - else: - assert got2 == alt_val - - # Set value to None - setattr(nv, name, None) - assert getattr(nv, name) is None - - -def test_nv_qldpc_decoder_config_setting_wrong_types_raises_typeerror(): - nv = qec.nv_qldpc_decoder_config() - - with pytest.raises(TypeError): - nv.max_iterations = "ten" - - with pytest.raises(TypeError): - nv.use_sparsity = "True" - - with pytest.raises(TypeError): - nv.error_rate = "0.1" - - with pytest.raises(TypeError): - nv.error_rate_vec = [0.1, "nope", 0.3] - - with pytest.raises(TypeError): - nv.error_rate_vec = 3.14 - - -def test_nv_qldpc_decoder_config_error_rate_vec_accepts_python_list_of_float(): - nv = qec.nv_qldpc_decoder_config() - - vals = [0.0, 0.125, 0.25] - nv.error_rate_vec = vals - got = nv.error_rate_vec - assert isinstance(got, list) - assert all(isinstance(x, float) for x in got) - assert got == vals - - -def test_nv_qldpc_decoder_config_toggle_multiple_fields_and_clear(): - nv = qec.nv_qldpc_decoder_config() - - nv.use_sparsity = True - nv.error_rate = 0.0123 - nv.error_rate_vec = [0.1, 0.2, 0.3] - nv.max_iterations = 100 - nv.n_threads = 8 - nv.use_osd = True - nv.osd_method = 2 - nv.osd_order = 4 - nv.bp_batch_size = 32 - nv.osd_batch_size = 16 - nv.iter_per_check = 3 - nv.clip_value = 7.5 - nv.bp_method = 1 - nv.scale_factor = 0.8 - nv.proc_float = "fp64" - - assert nv is not None - assert nv.use_sparsity is True - assert math.isclose(nv.error_rate, 0.0123) - assert nv.error_rate_vec == [0.1, 0.2, 0.3] - assert nv.max_iterations == 100 - assert nv.n_threads == 8 - - nv.use_sparsity = None - nv.error_rate = None - nv.error_rate_vec = None - nv.max_iterations = None - nv.n_threads = None - - assert nv.use_sparsity is None - assert nv.error_rate is None - assert nv.error_rate_vec is None - assert nv.max_iterations is None - assert nv.n_threads is None - - -# multi_error_lut_config tests - -FIELDS_MULTI_ERROR_LUT = { - "lut_error_depth": (int, 1, 3), -} - -# pymatching_config tests - -FIELDS_PYMATCHING = { - "error_rate_vec": (list, [0.1, 0.2, 0.3], [0.2, 0.1, 0.2]), - "merge_strategy": (str, "smallest_weight", "disallow"), -} - -FIELDS_CHROMOBIUS = { - "drop_mobius_errors_involving_remnant_errors": (bool, True, False), - "ignore_decomposition_failures": (bool, True, False), - "include_coords_in_mobius_dem": (bool, True, False), - "return_weight": (bool, True, False), - "write_mobius_match_to_stderr": (bool, True, False), -} - -# trt_decoder_config tests - -FIELDS_TRT_DECODER = { - "onnx_load_path": (str, "/path/to/model.onnx", "/other/path/model.onnx"), - "engine_load_path": (str, "/path/to/engine.trt", "/other/engine.trt"), - "engine_save_path": (str, "/path/to/save.trt", "/other/save.trt"), - "precision": (str, "fp16", "fp32"), - "memory_workspace": (int, 1073741824, 2147483648), # 1GB, 2GB -} - - -def test_multi_error_lut_config_defaults_are_none(): - m = qec.multi_error_lut_config() - for name in FIELDS_MULTI_ERROR_LUT: - assert getattr(m, name) is None, f"Expected {name} to default to None" - - -def test_pymatching_config_defaults_are_none(): - pm = qec.pymatching_config() - for name in FIELDS_PYMATCHING: - assert getattr(pm, name) is None, f"Expected {name} to default to None" - - -def test_chromobius_config_defaults_are_none(): - chromobius = qec.chromobius_config() - for name in FIELDS_CHROMOBIUS: - assert getattr(chromobius, name) is None - - -@pytest.mark.parametrize("name, meta", list(FIELDS_PYMATCHING.items())) -def test_pymatching_config_set_and_get_each_optional(name, meta): - pm = qec.pymatching_config() - - py_type, sample_val, alt_val = meta - - assert getattr(pm, name) is None - - setattr(pm, name, sample_val) - got = getattr(pm, name) - assert isinstance(got, py_type) - assert got == sample_val - - setattr(pm, name, alt_val) - got2 = getattr(pm, name) - assert got2 == alt_val - - setattr(pm, name, None) - assert getattr(pm, name) is None - - -@pytest.mark.parametrize("name, meta", list(FIELDS_CHROMOBIUS.items())) -def test_chromobius_config_set_and_get_each_optional(name, meta): - chromobius = qec.chromobius_config() - - py_type, sample_val, alt_val = meta - - assert getattr(chromobius, name) is None - - setattr(chromobius, name, sample_val) - got = getattr(chromobius, name) - assert isinstance(got, py_type) - assert got == sample_val - - setattr(chromobius, name, alt_val) - got2 = getattr(chromobius, name) - assert got2 == alt_val - - setattr(chromobius, name, None) - assert getattr(chromobius, name) is None - - -def test_configure_valid_multi_error_lut_decoders(): - nv = qec.multi_error_lut_config() - nv.lut_error_depth = 2 - - dc = qec.decoder_config() - dc.id = 0 - dc.type = "multi_error_lut" - dc.block_size = 10 - dc.syndrome_size = 3 - dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - dc.D_sparse = qec.generate_timelike_sparse_detector_matrix( - dc.syndrome_size, 2, include_first_round=False) - dc.set_decoder_custom_args(nv) - - mdc = qec.multi_decoder_config() - mdc.decoders = [dc] - ret = qec.configure_decoders(mdc) - qec.finalize_decoders() - assert isinstance(ret, int) - assert ret == 0 - - -def test_trt_decoder_config_defaults_are_none(): - trt = qec.trt_decoder_config() - for name in FIELDS_TRT_DECODER: - assert getattr(trt, name) is None, f"Expected {name} to default to None" - - -@pytest.mark.parametrize("name, meta", list(FIELDS_TRT_DECODER.items())) -def test_trt_decoder_config_set_and_get_each_optional(name, meta): - trt = qec.trt_decoder_config() - - py_type, sample_val, alt_val = meta - - # Initially None - assert getattr(trt, name) is None - - # Set to a valid value and get back - setattr(trt, name, sample_val) - got = getattr(trt, name) - assert isinstance(got, py_type) - assert got == sample_val - - # Change to an alternate valid value - setattr(trt, name, alt_val) - got2 = getattr(trt, name) - assert got2 == alt_val - - # Set value to None - setattr(trt, name, None) - assert getattr(trt, name) is None - - -@pytest.mark.skipif( - trt_schema_missing, - reason="trt_decoder plugin (and its parameter schema) not available") -def test_trt_decoder_config_yaml_roundtrip(): - trt = qec.trt_decoder_config() - trt.engine_load_path = "/path/to/engine.trt" - trt.precision = "fp16" - trt.memory_workspace = 1073741824 # 1GB - - dc = qec.decoder_config() - dc.id = 0 - dc.type = "trt_decoder" - dc.block_size = 10 - dc.syndrome_size = 3 - dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - dc.set_decoder_custom_args(trt) - - yaml_text = dc.to_yaml_str() - assert isinstance(yaml_text, str) and len(yaml_text) > 0 - - dc2 = qec.decoder_config.from_yaml_str(yaml_text) - - # Basic scalar fields - assert dc2 is not None - assert dc2.id == 0 - assert dc2.type == "trt_decoder" - assert dc2.block_size == 10 - assert dc2.syndrome_size == 3 - - # Intentional API change vs. the original test: decoder_custom_args now - # reads back as a plain dict, never a typed config object. - trt2 = dc2.decoder_custom_args - assert trt2 is not None - assert trt2["engine_load_path"] == "/path/to/engine.trt" - assert trt2["precision"] == "fp16" - assert trt2["memory_workspace"] == 1073741824 - - -def test_trt_decoder_config_chromobius_global_params_roundtrip(): - trt = qec.trt_decoder_config() - chromobius = qec.chromobius_config() - chromobius.return_weight = True - - trt.global_decoder = "chromobius" - trt.global_decoder_params = chromobius - - got = trt.global_decoder_params - assert isinstance(got, qec.chromobius_config) - assert got.return_weight is True - - as_map = trt.to_heterogeneous_map() - assert as_map["global_decoder"] == "chromobius" - assert as_map["global_decoder_params"]["return_weight"] is True - - trt2 = qec.trt_decoder_config.from_heterogeneous_map(as_map) - got2 = trt2.global_decoder_params - assert isinstance(got2, qec.chromobius_config) - assert got2.return_weight is True - - trt2.global_decoder_params = None - assert trt2.global_decoder_params is None - - -def test_trt_decoder_config_defaults_omitted_global_params(): - for global_decoder, config_type in ( - ("pymatching", qec.pymatching_config), - ("chromobius", qec.chromobius_config), - ): - trt = qec.trt_decoder_config.from_heterogeneous_map( - {"global_decoder": global_decoder}) - - got = trt.global_decoder_params - assert isinstance(got, config_type) - - as_map = trt.to_heterogeneous_map() - assert as_map["global_decoder"] == global_decoder - assert as_map["global_decoder_params"] == {} - - trt = qec.trt_decoder_config() - trt.global_decoder = global_decoder - as_map = trt.to_heterogeneous_map() - assert as_map["global_decoder"] == global_decoder - assert as_map["global_decoder_params"] == {} - - -def test_trt_decoder_config_preserves_unknown_omitted_global_params(): - trt = qec.trt_decoder_config.from_heterogeneous_map( - {"global_decoder": "my_plugin"}) - - assert trt.global_decoder_params is None - - as_map = trt.to_heterogeneous_map() - assert as_map["global_decoder"] == "my_plugin" - assert "global_decoder_params" not in as_map - - trt = qec.trt_decoder_config() - trt.global_decoder = "my_plugin" - as_map = trt.to_heterogeneous_map() - assert as_map["global_decoder"] == "my_plugin" - assert "global_decoder_params" not in as_map - - -def test_trt_decoder_config_rejects_unknown_global_params(): - with pytest.raises(RuntimeError): - qec.trt_decoder_config.from_heterogeneous_map({ - "global_decoder": "my_plugin", - "global_decoder_params": {}, - }) - - -def test_pymatching_config_yaml_roundtrip(): - pm = qec.pymatching_config() - pm.error_rate_vec = [0.1, 0.2, 0.3] - pm.merge_strategy = "smallest_weight" - - dc = qec.decoder_config() - dc.id = 0 - dc.type = "pymatching" - dc.block_size = 3 - dc.syndrome_size = 3 - dc.H_sparse = [0, -1, 1, -1, 2, -1] - dc.O_sparse = [0, -1, 1, -1, 2, -1] - dc.D_sparse = [0, -1, 1, -1, 2, -1] - dc.set_decoder_custom_args(pm) - - yaml_text = dc.to_yaml_str() - assert isinstance(yaml_text, str) and "pymatching" in yaml_text - - dc2 = qec.decoder_config.from_yaml_str(yaml_text) - assert dc2 is not None - assert dc2.type == "pymatching" - - # Intentional API change vs. the original test: decoder_custom_args now - # reads back as a plain dict, never a typed config object. - pm2 = dc2.decoder_custom_args - assert pm2 is not None - assert list(pm2["error_rate_vec"]) == [0.1, 0.2, 0.3] - assert pm2["merge_strategy"] == "smallest_weight" - - -@pytest.mark.skipif( - trt_schema_missing or chromobius_schema_missing, - reason="trt_decoder/chromobius plugins (and their schemas) not available") -def test_trt_decoder_chromobius_global_config_yaml_roundtrip(): - chromobius = qec.chromobius_config() - chromobius.ignore_decomposition_failures = True - chromobius.return_weight = False - - trt = qec.trt_decoder_config() - trt.global_decoder = "chromobius" - trt.global_decoder_params = chromobius - - dc = qec.decoder_config() - dc.id = 0 - dc.type = "trt_decoder" - dc.block_size = 3 - dc.syndrome_size = 3 - dc.H_sparse = [0, -1, 1, -1, 2, -1] - dc.O_sparse = [0, -1, 1, -1, 2, -1] - dc.D_sparse = [0, -1, 1, -1, 2, -1] - dc.set_decoder_custom_args(trt) - - yaml_text = dc.to_yaml_str() - assert isinstance(yaml_text, str) and "chromobius" in yaml_text - - dc2 = qec.decoder_config.from_yaml_str(yaml_text) - assert dc2 is not None - assert dc2.type == "trt_decoder" - - # Intentional API change vs. the original test: decoder_custom_args now - # reads back as a plain dict, never a typed config object. - trt2 = dc2.decoder_custom_args - assert trt2 is not None - assert trt2["global_decoder"] == "chromobius" - - chromobius2 = trt2["global_decoder_params"] - assert chromobius2 is not None - assert chromobius2["ignore_decomposition_failures"] is True - assert chromobius2["return_weight"] is False - - -# decoder_config tests - - -def test_decoder_config_yaml_roundtrip_and_custom_args(): - # Build NV config and embed into DecoderConfig via helper - nv = qec.nv_qldpc_decoder_config() - nv.use_sparsity = True - nv.error_rate = 0.01 - nv.max_iterations = 50 - nv.error_rate_vec = [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1] - - dc = qec.decoder_config() - dc.id = 0 - dc.type = "nv-qldpc-decoder" - dc.block_size = 10 - dc.syndrome_size = 3 - dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - dc.set_decoder_custom_args(nv) - - yaml_text = dc.to_yaml_str() - assert isinstance(yaml_text, str) and len(yaml_text) > 0 - - dc2 = qec.decoder_config.from_yaml_str(yaml_text) - - # Basic scalar fields - assert dc2 is not None - assert dc2.id == 0 - assert dc2.type == "nv-qldpc-decoder" - assert dc2.block_size == 10 - assert dc2.syndrome_size == 3 - - # Intentional API change vs. the original test: decoder_custom_args now - # reads back as a plain dict, never a typed config object. - nv2 = dc2.decoder_custom_args - assert nv2 is not None - assert nv2["use_sparsity"] is True - assert math.isclose(nv2["error_rate"], 0.01) - assert nv2["max_iterations"] == 50 - - -# multi_decoder_config tests - - -def test_multi_decoder_config_yaml_roundtrip(): - # Build NV config and embed into DecoderConfig via helper - nv = qec.nv_qldpc_decoder_config() - nv.use_sparsity = True - nv.error_rate = 0.01 - nv.error_rate_vec = [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1] - nv.max_iterations = 50 - - d1 = qec.decoder_config() - d1.id = 0 - d1.type = "nv-qldpc-decoder" - d1.block_size = 10 - d1.syndrome_size = 3 - d1.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - d1.set_decoder_custom_args(nv) - - lut_config = qec.multi_error_lut_config() - lut_config.lut_error_depth = 3 - - d2 = qec.decoder_config() - d2.id = 1 - d2.type = "multi_error_lut" - d2.block_size = 10 - d2.syndrome_size = 3 - d2.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - d2.set_decoder_custom_args(lut_config) - - mdc = qec.multi_decoder_config() - mdc.decoders = [d1, d2] - - yaml_text = mdc.to_yaml_str() - assert isinstance(yaml_text, str) and "0" in yaml_text and "1" in yaml_text - - mdc2 = qec.multi_decoder_config.from_yaml_str(yaml_text) - assert mdc2 is not None - assert len(mdc2.decoders) == 2 - ids = sorted({md.id for md in mdc2.decoders}) - assert ids == [0, 1] - - -def test_configure_decoders_from_str_smoke(): - multi_decoder_config = qec.multi_decoder_config() - yaml_str = multi_decoder_config.to_yaml_str() - status = qec.configure_decoders_from_str(yaml_str) - assert isinstance(status, int) - qec.finalize_decoders() - - nv = qec.nv_qldpc_decoder_config() - nv.error_rate_vec = [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1] - - decoder_config = qec.decoder_config() - decoder_config.id = 0 - decoder_config.type = "nv-qldpc-decoder" - decoder_config.block_size = 10 - decoder_config.syndrome_size = 3 - decoder_config.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - decoder_config.set_decoder_custom_args(nv) - multi_decoder_config = qec.multi_decoder_config() - multi_decoder_config.decoders = [decoder_config] - yaml_str = multi_decoder_config.to_yaml_str() - # Do not instantiate the decoder if it is not available. - if not is_nv_qldpc_decoder_available(): - return - status = qec.configure_decoders_from_str(yaml_str) - assert isinstance(status, int) - qec.finalize_decoders() - - -def test_configure_valid_decoders(): - nv = qec.nv_qldpc_decoder_config() - nv.use_sparsity = True - nv.error_rate = 0.01 - nv.error_rate_vec = [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1] - nv.max_iterations = 50 - - dc = qec.decoder_config() - dc.id = 0 - dc.type = "multi_error_lut" - dc.block_size = 10 - dc.syndrome_size = 3 - dc.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - dc.D_sparse = qec.generate_timelike_sparse_detector_matrix( - dc.syndrome_size, 2, include_first_round=False) - lut_config = qec.multi_error_lut_config() - lut_config.lut_error_depth = 2 - dc.set_decoder_custom_args(lut_config) - - mdc = qec.multi_decoder_config() - mdc.decoders = [dc] - ret = qec.configure_decoders(mdc) - qec.finalize_decoders() - assert isinstance(ret, int) - assert ret == 0 - - -def make_pymatching_multi_decoder_config(pm, h_sparse=None): - dc = qec.decoder_config() - dc.id = 0 - dc.type = "pymatching" - dc.block_size = 3 - dc.syndrome_size = 3 - dc.H_sparse = h_sparse if h_sparse is not None else [0, -1, 1, -1, 2, -1] - dc.O_sparse = [0, -1, 1, -1, 2, -1] - dc.D_sparse = [0, -1, 1, -1, 2, -1] - dc.set_decoder_custom_args(pm) - - mdc = qec.multi_decoder_config() - mdc.decoders = [dc] - return mdc - - -def configure_pymatching_status(pm, h_sparse=None): - try: - return qec.configure_decoders( - make_pymatching_multi_decoder_config(pm, h_sparse)) - finally: - qec.finalize_decoders() - - -def test_configure_valid_pymatching_decoder(): - pm = qec.pymatching_config() - pm.error_rate_vec = [0.1, 0.1, 0.1] - pm.merge_strategy = "smallest_weight" - - ret = configure_pymatching_status(pm) - assert isinstance(ret, int) - assert ret == 0 - - -@pytest.mark.parametrize( - "error_rate_vec", - ([0.1, 0.1], [0.0, 0.1, 0.1], [0.1, 0.6, 0.1]), -) -def test_configure_invalid_pymatching_error_rate_vec(error_rate_vec): - pm = qec.pymatching_config() - pm.error_rate_vec = error_rate_vec - pm.merge_strategy = "smallest_weight" - - ret = configure_pymatching_status(pm) - assert isinstance(ret, int) - assert ret != 0 - - -def test_configure_invalid_pymatching_merge_strategy(): - pm = qec.pymatching_config() - pm.error_rate_vec = [0.1, 0.1, 0.1] - pm.merge_strategy = "not-a-strategy" - - ret = configure_pymatching_status(pm) - assert isinstance(ret, int) - assert ret != 0 - - -def test_configure_invalid_pymatching_non_graphlike_h_sparse(): - pm = qec.pymatching_config() - pm.error_rate_vec = [0.1, 0.1, 0.1] - pm.merge_strategy = "smallest_weight" - - ret = configure_pymatching_status(pm, h_sparse=[0, -1, 0, -1, 0, -1]) - assert isinstance(ret, int) - assert ret != 0 - - -def test_configure_invalid_decoders(): - nv = qec.nv_qldpc_decoder_config() - nv.use_sparsity = True - nv.error_rate = 0.01 - nv.error_rate_vec = [0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1, 0.2, 0.3, 0.1] - nv.max_iterations = 50 - - decoder_config = qec.decoder_config() - decoder_config.id = 0 - decoder_config.type = "invalid-decoder" - decoder_config.block_size = 10 - decoder_config.syndrome_size = 3 - decoder_config.H_sparse = [1, 2, 3, -1, 6, 7, 8, -1, -1] - decoder_config.set_decoder_custom_args(nv) - - multi_decoder_config = qec.multi_decoder_config() - multi_decoder_config.decoders = [decoder_config] - ret = qec.configure_decoders(multi_decoder_config) - assert isinstance(ret, int) - assert ret != 0 - - -if __name__ == "__main__": - pytest.main() diff --git a/libs/qec/tools/CMakeLists.txt b/libs/qec/tools/CMakeLists.txt deleted file mode 100644 index fe875d614..000000000 --- a/libs/qec/tools/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# ============================================================================ # -# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # -# All rights reserved. # -# # -# This source code and the accompanying materials are made available under # -# the terms of the Apache License 2.0 which accompanies this distribution. # -# ============================================================================ # -add_subdirectory(decoding-server) diff --git a/libs/qec/tools/decoding-server/CMakeLists.txt b/libs/qec/tools/decoding-server/CMakeLists.txt deleted file mode 100644 index a9c678dc7..000000000 --- a/libs/qec/tools/decoding-server/CMakeLists.txt +++ /dev/null @@ -1,211 +0,0 @@ -# ============================================================================ # -# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # -# All rights reserved. # -# # -# This source code and the accompanying materials are made available under # -# the terms of the Apache License 2.0 which accompanies this distribution. # -# ============================================================================ # - -# decoding_server is a decoder-agnostic decoding-server process: the -# service end of a CUDA-Q device_call transport, wiring the libcudaq-realtime -# HOST_CALL dispatcher onto a transceiver's rings and serving the -# decoding-server-cqr function table (enqueue_syndromes / get_corrections / -# reset_decoder). Its decoders are selected entirely by a YAML config file -# (see decoding_server_config.yaml) and its transport by --transport=udp -# (loopback-capable, runs anywhere) or --transport=cpu_roce (real RDMA wire, -# compiled in when libcudaq-realtime-cpu-roce-transport + ibverbs are found). -# -# The two-process test (unittests/realtime/test_decoding_server.cpp) spawns -# this binary as a subprocess. - -if(CUDAQ_REALTIME_INCLUDE_DIR AND TARGET cudaq-qec-realtime-decoding-server-cqr) - # The server needs the DeviceCallService plugin headers, the libcudaq-realtime - # shared lib (cudaq_host_dispatcher_loop lives there, device-linked), and the - # cpu_transport UDP ring transceiver static lib. - find_path(QEC_UDP_DEVICE_CALL_INCLUDE_DIR - NAMES cudaq/realtime/device_call_service.h - HINTS ${CUDAQ_REALTIME_ROOT} ${CUDAQ_INSTALL_DIR} - PATH_SUFFIXES include) - find_library(QEC_UDP_REALTIME_LIBRARY cudaq-realtime - HINTS ${CUDAQ_REALTIME_ROOT} ${CUDAQ_INSTALL_DIR} - PATH_SUFFIXES lib) - find_library(QEC_UDP_TRANSPORT_LIBRARY cudaq-realtime-udp-transport - HINTS ${CUDAQ_REALTIME_ROOT} ${CUDAQ_INSTALL_DIR} - PATH_SUFFIXES lib) - # The host-side ring dispatch loop (cudaq_host_ring_dispatch_loop) lives in - # its own static archive since the realtime host-dispatch split; it is no - # longer part of libcudaq-realtime. - find_library(QEC_HOST_DISPATCH_LIBRARY cudaq-realtime-host-dispatch - HINTS ${CUDAQ_REALTIME_ROOT} ${CUDAQ_INSTALL_DIR} - PATH_SUFFIXES lib) - # Optional: the server's --transport=cpu_roce path (real RDMA wire) needs - # the CPU RoCE transceiver and libibverbs; without them the server still - # builds with the udp transport only. - find_library(QEC_CPU_ROCE_TRANSPORT_LIBRARY cudaq-realtime-cpu-roce-transport - HINTS ${CUDAQ_REALTIME_ROOT} ${CUDAQ_INSTALL_DIR} - PATH_SUFFIXES lib) - find_library(QEC_IBVERBS_LIBRARY ibverbs) - - if(QEC_UDP_DEVICE_CALL_INCLUDE_DIR AND QEC_UDP_REALTIME_LIBRARY - AND QEC_UDP_TRANSPORT_LIBRARY AND QEC_HOST_DISPATCH_LIBRARY) - add_executable(decoding_server decoding_server.cpp) - target_include_directories(decoding_server PRIVATE - ${CUDAQ_REALTIME_INCLUDE_DIR} - ${QEC_UDP_DEVICE_CALL_INCLUDE_DIR} - ${CUDAToolkit_INCLUDE_DIRS} - ) - target_link_directories(decoding_server - PRIVATE ${CUDAQ_INSTALL_DIR}/lib) - target_link_libraries(decoding_server PRIVATE - cudaq-qec - cudaq-qec-realtime-decoding - cudaq-qec-realtime-decoding-server-cqr - ${QEC_UDP_REALTIME_LIBRARY} - ${QEC_UDP_TRANSPORT_LIBRARY} - ${QEC_HOST_DISPATCH_LIBRARY} - cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime - nvqir nvqir-stim - CUDA::cudart - ) - if(QEC_CPU_ROCE_TRANSPORT_LIBRARY AND QEC_IBVERBS_LIBRARY) - target_compile_definitions(decoding_server PRIVATE - QEC_HAVE_CPU_ROCE_TRANSPORT) - target_link_libraries(decoding_server PRIVATE - ${QEC_CPU_ROCE_TRANSPORT_LIBRARY} - ${QEC_IBVERBS_LIBRARY}) - message(STATUS "decoding_server: cpu_roce transport enabled") - endif() - # The gpu_roce CLI branch ([2a] in decoding_server.cpp) references only - # the core DecodingServer API, so compile it whenever hololink/DOCA are - # present -- machines without the proprietary archive (CI in particular) - # still build the branch; at runtime it fails with a clear "GPU RoCE - # support is not linked into this binary" error from make_transport. - if(CUDAQ_GPU_ROCE_AVAILABLE) - target_compile_definitions(decoding_server PRIVATE - QEC_HAVE_GPU_ROCE_TRANSPORT) - target_include_directories(decoding_server PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/realtime/decoding-server-cqr - ) - target_link_libraries(decoding_server PRIVATE - cudaq-qec-decoding-server) - endif() - # Actually LINKING the GPU RoCE transport needs BOTH the - # cudaq-qec-decoding-server-gpuroce component AND the proprietary - # cudevice archive, which supplies the three populate-shim symbols - # (cudaqx_qec_realtime_dispatch_populate_*_device_entry) - # that GpuRoceTransceiver::launch_scheduler resolves via dlsym. Hololink - # presence and the proprietary archive are provisioned independently, so - # treat a missing archive as "gpu_roce transport not available" and fall - # back to the udp/cpu_roce transports rather than failing configuration. - set(_decoding_server_linked_proprietary_cudevice FALSE) - if(CUDAQ_GPU_ROCE_AVAILABLE AND TARGET cudaq-qec-realtime-cudevice-proprietary) - # Re-find in case the CQR subdir cache entry didn't propagate here. - if(NOT CUDAQ_REALTIME_DISPATCH_LIBRARY) - find_library(CUDAQ_REALTIME_DISPATCH_LIBRARY cudaq-realtime-dispatch - HINTS ${CUDAQ_REALTIME_ROOT} ${CUDAQ_INSTALL_DIR} - PATH_SUFFIXES lib) - endif() - if(NOT CUDAQ_REALTIME_DISPATCH_LIBRARY) - message(FATAL_ERROR - "CUDAQ_GPU_ROCE_AVAILABLE is set but cudaq-realtime-dispatch was " - "not found (searched CUDAQ_REALTIME_ROOT=${CUDAQ_REALTIME_ROOT} and " - "CUDAQ_INSTALL_DIR=${CUDAQ_INSTALL_DIR}). " - "GpuRoceTransceiver::launch_scheduler resolves " - "cudaq_create/launch/destroy_dispatch_graph_regular via dlsym; " - "without this library in the server image those lookups will fail " - "at runtime. Set CUDAQ_REALTIME_ROOT to the CUDA-Q realtime " - "install prefix and re-run CMake.") - endif() - target_link_libraries(decoding_server PRIVATE - # WHOLE_ARCHIVE: DecodingServer reaches gpu_roce through a weak - # factory symbol, and a weak reference does not pull archive members. - $ - ${CUDAQ_REALTIME_DISPATCH_LIBRARY} - $ - ) - set(_decoding_server_linked_proprietary_cudevice TRUE) - target_link_options(decoding_server PRIVATE - "LINKER:--export-dynamic" - ) - message(STATUS "decoding_server: gpu_roce transport enabled") - elseif(CUDAQ_GPU_ROCE_AVAILABLE) - message(WARNING - "decoding_server: gpu_roce CLI compiled but transport NOT linked " - "(cudaq-qec-realtime-cudevice-proprietary target not defined; set " - "CUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE to enable). " - "--transport=gpu_roce will fail at runtime with a not-linked error.") - endif() - # RelayBP/nv-qldpc captures a CUDA device graph during decoder - # initialization even when the server transport is udp or cpu_roce. Export - # the graph-dispatch shims whenever a cudevice archive is available so - # plugin dlsym(RTLD_DEFAULT, ...) lookups succeed. The proprietary archive - # also carries the public gpu_kernels.cu.o symbols, so do not whole-archive - # both the proprietary and public cudevice archives into the same binary. - if(TARGET cudaq-qec-realtime-cudevice OR TARGET cudaq-qec-realtime-cudevice-proprietary) - target_link_libraries(decoding_server PRIVATE - CUDA::cudart - ) - if(TARGET cudaq-qec-realtime-cudevice-proprietary) - if(NOT _decoding_server_linked_proprietary_cudevice) - target_link_libraries(decoding_server PRIVATE - $ - ) - endif() - elseif(TARGET cudaq-qec-realtime-cudevice) - target_link_libraries(decoding_server PRIVATE - $ - ) - endif() - target_link_options(decoding_server PRIVATE - "LINKER:--export-dynamic" - ) - set_property(TARGET decoding_server PROPERTY CUDA_SEPARABLE_COMPILATION ON) - set_property(TARGET decoding_server PROPERTY CUDA_RESOLVE_DEVICE_SYMBOLS ON) - set_property(TARGET decoding_server PROPERTY CUDA_STANDARD 17) - set_property(TARGET decoding_server PROPERTY LINKER_LANGUAGE CUDA) - endif() - # The installed binary lives in /bin with the qec libraries and - # decoder plugins in ../lib, so use $ORIGIN-relative rpaths for those; - # the CUDA-Q realtime libs stay at their absolute install location. - get_filename_component(_decoding_server_realtime_lib_dir - "${QEC_UDP_REALTIME_LIBRARY}" DIRECTORY) - set_target_properties(decoding_server PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin - BUILD_RPATH - "${CMAKE_BINARY_DIR}/lib;${CMAKE_BINARY_DIR}/lib/decoder-plugins;${CUDAQ_INSTALL_DIR}/lib" - INSTALL_RPATH - "$ORIGIN/../lib;$ORIGIN/../lib/decoder-plugins;${_decoding_server_realtime_lib_dir};${CUDAQ_INSTALL_DIR}/lib" - ) - # Decoder plugins are loaded at runtime by decoder type name; the example - # configs currently select pymatching / multi_error_lut. - add_dependencies(decoding_server cudaq-qec-pymatching) - - # Place the example decoder configs next to the server binary. - configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/decoding_server_config.yaml - ${CMAKE_BINARY_DIR}/bin/decoding_server_config.yaml - COPYONLY) - configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/decoding_server_config_multi_error_lut.yaml - ${CMAKE_BINARY_DIR}/bin/decoding_server_config_multi_error_lut.yaml - COPYONLY) - - # Install the server (and its example configs) alongside the libraries. - # The qec-tools component is deliberately NOT in the Python wheel's - # install.components (pyproject.toml): the server depends on the CUDA-Q - # realtime libs, which the cudaq wheel does not ship. - install(TARGETS decoding_server - COMPONENT qec-tools - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - ) - install(FILES - ${CMAKE_CURRENT_SOURCE_DIR}/decoding_server_config.yaml - ${CMAKE_CURRENT_SOURCE_DIR}/decoding_server_config_multi_error_lut.yaml - COMPONENT qec-tools - DESTINATION ${CMAKE_INSTALL_BINDIR} - ) - else() - message(STATUS "decoding_server: cudaq-realtime udp transport not " - "found; decoding-server tool will not be built.") - endif() -endif() diff --git a/libs/qec/tools/decoding-server/decoding_server.cpp b/libs/qec/tools/decoding-server/decoding_server.cpp deleted file mode 100644 index 09ace7c79..000000000 --- a/libs/qec/tools/decoding-server/decoding_server.cpp +++ /dev/null @@ -1,706 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -/// @file decoding_server.cpp -/// @brief Standalone decoding-server process: the service end of a -/// CUDA-Q device_call transport, decoding on the CPU with whatever decoder a -/// YAML config file selects. -/// -/// This is the two-process analogue of the in-process host_dispatch device -/// call tests, structured exactly like CUDA-Q's cpu_roce_test_daemon: a -/// cpu_transport transceiver owns the wire and the rings, and -/// libcudaq-realtime's HOST_CALL host-dispatcher loop is wired straight onto -/// those rings. Both the decoder and the transport are configuration, not -/// code: -/// - decoders come from `--config=` -/// (multi_decoder_config::from_yaml_str); -/// - the transport comes from `--transport=udp|cpu_roce`: the UDP ring -/// transceiver (loopback; runs anywhere) or the CPU RoCE RDMA transceiver -/// (requires an RDMA NIC; pairs with the caller's -/// `--cudaq-device-call=cpu_roce` channel and includes the QP/rkey TCP -/// rendezvous server). -/// - for cpu_roce, `--qp_config=rendezvous|hsb_fpga` selects how queue pairs -/// are exchanged. `rendezvous` (default) is the TCP QP/rkey swap with a -/// CpuRoceChannel caller. `hsb_fpga` is the Holoscan-Sensor-Bridge FPGA -/// method: the peer QP comes from `--remote-qp` (the FPGA data-plane QP, -/// or the emulator's QP) and this server prints its own QP / RKey / -/// Buffer Addr in the canonical bridge handshake format -/// (hololink_bridge_common.h) for the orchestration script to relay to -/// the playback tool -- which alone programs the FPGA over the Hololink -/// control plane. The server itself performs NO control-plane traffic. -/// -/// The function table comes from the decoding-server-cqr service plugin -/// (enqueue_syndromes / get_corrections / reset_decoder) regardless of -/// transport or decoder. -/// -/// Prints `QEC_DECODING_SERVER_READY port=

...` on stdout once listening -/// (for udp, P is the UDP port; for cpu_roce, P is the TCP rendezvous port and -/// the line also carries `roce_ip=`), and -/// `QEC_DECODING_SERVER_DISPATCHED count=` at shutdown (the two-process -/// stand-in for the in-process cudaqx_qec_device_call_dispatch_count() -/// assertion). -/// -/// Usage: -/// decoding_server --config= -/// [--transport=udp|cpu_roce] [--port=0] -/// [--num-slots=8] [--slot-size=256] [--timeout=60] -/// [--device=mlx5_0] [--local-ip=10.0.0.2] -/// [--qp_config=rendezvous|hsb_fpga] -/// [--peer-ip=ADDR] [--remote-qp=0x2] -/// [--frame-size=N] -/// -/// NOTE: --slot-size must match the caller channel's slot size (each frame -/// occupies one full slot stride on both wires). With --qp_config=hsb_fpga, -/// --slot-size is the HSB page size (ring slot stride) and --num-slots is -/// capped at 64 (the HSB WQE depth). - -#include "cudaq/qec/realtime/decoding_config.h" - -#include "cudaq/realtime/device_call_service.h" - -#include "cudaq/realtime/cpu_transport/udp_wrapper.h" -#include "cudaq/realtime/daemon/dispatcher/cudaq_realtime.h" -#include "cudaq/realtime/daemon/dispatcher/dispatch_kernel_launch.h" -#include "cudaq/realtime/daemon/dispatcher/graph_launch_engine.h" - -#ifdef QEC_HAVE_CPU_ROCE_TRANSPORT -#include "cudaq/realtime/cpu_transport/roce_wrapper.h" - -#include -#include -#include -#include -#include -#endif - -#ifdef QEC_HAVE_GPU_ROCE_TRANSPORT -// DecodingServer.h (and GpuRoceTransceiver.h via DecodingServer.cpp) live in -// the decoding-server-cqr directory, added to include paths by CMakeLists when -// CUDAQ_GPU_ROCE_AVAILABLE is true. -#include "DecodingServer.h" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" void cudaqx_qec_realtime_device_call_service_force_link(); -extern "C" std::uint64_t cudaqx_qec_device_call_dispatch_count(); -extern "C" std::uint64_t cudaqx_qec_decoding_server_max_concurrent(); -extern "C" void cudaqx_qec_decoding_server_print_stats(); -extern "C" void cudaqx_qec_decoding_server_shutdown(); - -namespace { - -namespace config = cudaq::qec::decoding::config; - -struct ServerConfig { - std::string config_path; - std::string transport = "udp"; - std::uint16_t port = 0; // 0 => ephemeral, printed on stdout - std::uint32_t num_slots = 8; - std::size_t slot_size = 256; - int timeout_sec = 60; - // cpu_roce only: - std::string device = "mlx5_0"; - std::string local_ip = "10.0.0.2"; - // cpu_roce QP exchange method (see file header). - std::string qp_config = "rendezvous"; - // hsb_fpga only: - std::string peer_ip; // FPGA/emulator data-plane IPv4 (required) - std::uint32_t remote_qp = 0x2; // FPGA data-plane QP (emulator QP in emulate) - std::size_t frame_size = 0; // TX SGE bytes; 0 => slot_size -}; - -bool starts_with(const std::string &s, const char *prefix) { - const std::size_t n = std::strlen(prefix); - return s.size() >= n && std::memcmp(s.data(), prefix, n) == 0; -} - -bool parse_args(int argc, char **argv, ServerConfig &cfg) { - for (int i = 1; i < argc; ++i) { - const std::string a = argv[i]; - if (a == "--help" || a == "-h") { - std::cout << "Usage: " << argv[0] - << " --config= " - "[--transport=udp|cpu_roce|gpu_roce] " - "[--port=N] [--num-slots=N] [--slot-size=N] [--timeout=N] " - "[--device=NAME] [--local-ip=ADDR] " - "[--qp_config=rendezvous|hsb_fpga] [--peer-ip=ADDR] " - "[--remote-qp=N] [--frame-size=N]" - << std::endl; - return false; - } else if (starts_with(a, "--config=")) - cfg.config_path = a.substr(9); - else if (starts_with(a, "--transport=")) - cfg.transport = a.substr(12); - else if (starts_with(a, "--port=")) - cfg.port = static_cast(std::stoul(a.substr(7))); - else if (starts_with(a, "--num-slots=")) - cfg.num_slots = static_cast(std::stoul(a.substr(12))); - else if (starts_with(a, "--slot-size=")) - cfg.slot_size = std::stoull(a.substr(12)); - else if (starts_with(a, "--timeout=")) - cfg.timeout_sec = std::stoi(a.substr(10)); - else if (starts_with(a, "--device=")) - cfg.device = a.substr(9); - else if (starts_with(a, "--local-ip=")) - cfg.local_ip = a.substr(11); - else if (starts_with(a, "--qp_config=")) - cfg.qp_config = a.substr(12); - else if (starts_with(a, "--peer-ip=")) - cfg.peer_ip = a.substr(10); - else if (starts_with(a, "--remote-qp=")) - // base 0: accepts both decimal and 0x-prefixed hex (QP numbers are - // conventionally printed in hex, e.g. the FPGA's fixed 0x2). - cfg.remote_qp = - static_cast(std::stoul(a.substr(12), nullptr, 0)); - else if (starts_with(a, "--frame-size=")) - cfg.frame_size = std::stoull(a.substr(13)); - else { - std::cerr << "Unknown argument: " << a << " (use --help)" << std::endl; - return false; - } - } - if (cfg.config_path.empty()) { - std::cerr << "ERROR: --config= is required" << std::endl; - return false; - } - if (cfg.qp_config != "rendezvous" && cfg.qp_config != "hsb_fpga") { - std::cerr << "ERROR: unknown --qp_config=" << cfg.qp_config - << " (expected rendezvous or hsb_fpga)" << std::endl; - return false; - } - if (cfg.qp_config == "hsb_fpga") { - if (cfg.transport != "cpu_roce") { - std::cerr << "ERROR: --qp_config=hsb_fpga requires --transport=cpu_roce" - << std::endl; - return false; - } - if (cfg.peer_ip.empty()) { - std::cerr << "ERROR: --qp_config=hsb_fpga requires --peer-ip=" - << std::endl; - return false; - } - // The HSB receive queue is WQE_NUM=64 deep; a deeper ring would alias two - // slots per WQE and race RX against TX (same constraint as the Hololink - // bridges). - constexpr std::uint32_t kHsbWqeNum = 64; - if (cfg.num_slots > kHsbWqeNum) { - std::cerr << "WARNING: --num-slots=" << cfg.num_slots << " exceeds the " - << "HSB WQE depth; clamping to " << kHsbWqeNum << std::endl; - cfg.num_slots = kHsbWqeNum; - } - } - return true; -} - -std::atomic g_shutdown{0}; -void on_signal(int) { g_shutdown.store(1, std::memory_order_release); } - -// Transport-agnostic view of one wired-up transceiver: the four ring -// addresses the dispatcher consumes, plus a teardown hook. Both transports -// provide the identical ring contract (see udp_wrapper.h / roce_wrapper.h). -struct TransportEndpoints { - volatile std::uint64_t *rx_flags = nullptr; - volatile std::uint64_t *tx_flags = nullptr; - std::uint8_t *rx_data = nullptr; - std::uint8_t *tx_data = nullptr; - std::function shutdown; -}; - -// Publish the rendezvous endpoint for the test fixture. Emitted once the -// caller can start connecting (udp: socket bound; cpu_roce: TCP rendezvous -// listening). -void print_ready(std::uint16_t port, const std::string &extra) { - std::cout << "QEC_DECODING_SERVER_READY port=" << port - << (extra.empty() ? "" : " ") << extra << std::endl; - std::cout.flush(); -} - -bool init_udp_transport(const ServerConfig &cfg, TransportEndpoints &tp) { - cpu_udp_transceiver_t xcvr = - cpu_udp_create_transceiver(cfg.slot_size, cfg.num_slots); - if (!xcvr) { - std::cerr << "ERROR: udp transceiver create failed" << std::endl; - return false; - } - if (!cpu_udp_bind(xcvr, cfg.port) || !cpu_udp_start(xcvr)) { - std::cerr << "ERROR: udp transceiver bind/start failed" << std::endl; - cpu_udp_destroy_transceiver(xcvr); - return false; - } - tp.rx_flags = reinterpret_cast( - cpu_udp_get_rx_ring_flag_addr(xcvr)); - tp.tx_flags = reinterpret_cast( - cpu_udp_get_tx_ring_flag_addr(xcvr)); - tp.rx_data = - reinterpret_cast(cpu_udp_get_rx_ring_data_addr(xcvr)); - tp.tx_data = - reinterpret_cast(cpu_udp_get_tx_ring_data_addr(xcvr)); - tp.shutdown = [xcvr] { - cpu_udp_close(xcvr); - cpu_udp_destroy_transceiver(xcvr); - }; - print_ready(cpu_udp_get_port(xcvr), "transport=udp"); - return true; -} - -#ifdef QEC_HAVE_CPU_ROCE_TRANSPORT - -// Must match CpuRoceChannel's RendezvousInfo byte-for-byte (network order). -struct RendezvousInfo { - std::uint32_t qp_number = 0; - std::uint32_t rkey = 0; - std::uint32_t roce_ipv4 = 0; -}; - -bool write_all(int fd, const void *buf, std::size_t len) { - const auto *p = static_cast(buf); - while (len > 0) { - const ssize_t n = ::write(fd, p, len); - if (n <= 0) { - if (n < 0 && errno == EINTR) - continue; - return false; - } - p += n; - len -= static_cast(n); - } - return true; -} - -bool read_all(int fd, void *buf, std::size_t len) { - auto *p = static_cast(buf); - while (len > 0) { - const ssize_t n = ::read(fd, p, len); - if (n <= 0) { - if (n < 0 && errno == EINTR) - continue; - return false; - } - p += n; - len -= static_cast(n); - } - return true; -} - -// Service-end CPU RoCE bring-up, mirroring cpu_roce_test_daemon: transceiver -// setup, TCP rendezvous server (READY printed once listening; blocks in -// accept until the caller channel connects), QP/rkey swap, connect, monitor -// thread. tx_mode=RDMA_SEND: we Send responses; the caller Writes requests. -bool init_cpu_roce_transport(const ServerConfig &cfg, TransportEndpoints &tp) { - cpu_roce_transceiver_t xcvr = cpu_roce_create_transceiver( - cfg.device.c_str(), /*ib_port=*/1, /*tx_ibv_qp=*/0u, - /*frame_size=*/cfg.slot_size, /*page_size=*/cfg.slot_size, cfg.num_slots, - /*peer_ip=*/"0.0.0.0", /*forward=*/0, /*rx_only=*/0, /*tx_only=*/0, - /*unified=*/0, CPU_ROCE_TX_MODE_RDMA_SEND, /*peer_rx_base_addr=*/0, - /*peer_rx_rkey=*/0); - if (!xcvr) { - std::cerr << "ERROR: cpu_roce transceiver create failed" << std::endl; - return false; - } - cpu_roce_set_local_ip(xcvr, cfg.local_ip.c_str()); - if (!cpu_roce_setup(xcvr)) { - std::cerr << "ERROR: cpu_roce transceiver setup() failed" << std::endl; - cpu_roce_destroy_transceiver(xcvr); - return false; - } - - // TCP rendezvous server: mirror of CpuRoceChannel::exchangeRendezvous - // (server reads the caller's {qp, rkey, ip} first, then replies). - const int listen_fd = ::socket(AF_INET, SOCK_STREAM, 0); - if (listen_fd < 0) { - std::cerr << "ERROR: rendezvous socket() failed" << std::endl; - cpu_roce_destroy_transceiver(xcvr); - return false; - } - int reuse = 1; - ::setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); - sockaddr_in srv{}; - srv.sin_family = AF_INET; - srv.sin_addr.s_addr = htonl(INADDR_ANY); - srv.sin_port = htons(cfg.port); - if (::bind(listen_fd, reinterpret_cast(&srv), sizeof(srv)) != 0 || - ::listen(listen_fd, 1) != 0) { - std::cerr << "ERROR: rendezvous bind/listen failed" << std::endl; - ::close(listen_fd); - cpu_roce_destroy_transceiver(xcvr); - return false; - } - socklen_t srvlen = sizeof(srv); - ::getsockname(listen_fd, reinterpret_cast(&srv), &srvlen); - print_ready(ntohs(srv.sin_port), - "transport=cpu_roce roce_ip=" + cfg.local_ip); - - const int conn_fd = ::accept(listen_fd, nullptr, nullptr); - ::close(listen_fd); - if (conn_fd < 0) { - std::cerr << "ERROR: rendezvous accept() failed" << std::endl; - cpu_roce_destroy_transceiver(xcvr); - return false; - } - int one = 1; - ::setsockopt(conn_fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); - - RendezvousInfo peer{}; - in_addr local_addr{}; - ::inet_pton(AF_INET, cfg.local_ip.c_str(), &local_addr); - const RendezvousInfo self{htonl(cpu_roce_get_qp_number(xcvr)), - htonl(cpu_roce_get_rkey(xcvr)), local_addr.s_addr}; - if (!read_all(conn_fd, &peer, sizeof(peer)) || - !write_all(conn_fd, &self, sizeof(self))) { - std::cerr << "ERROR: rendezvous exchange failed" << std::endl; - ::close(conn_fd); - cpu_roce_destroy_transceiver(xcvr); - return false; - } - ::close(conn_fd); - - char peer_ip[INET_ADDRSTRLEN] = {0}; - in_addr peer_addr{}; - peer_addr.s_addr = peer.roce_ipv4; - ::inet_ntop(AF_INET, &peer_addr, peer_ip, sizeof(peer_ip)); - // We Send responses (no RDMA Writes to the caller), so no peer rkey needed. - if (!cpu_roce_connect(xcvr, ntohl(peer.qp_number), peer_ip, - /*peer_rx_rkey=*/0)) { - std::cerr << "ERROR: cpu_roce transceiver connect() failed" << std::endl; - cpu_roce_destroy_transceiver(xcvr); - return false; - } - - auto *monitor = new std::thread([xcvr] { cpu_roce_blocking_monitor(xcvr); }); - - tp.rx_flags = reinterpret_cast( - cpu_roce_get_rx_ring_flag_addr(xcvr)); - tp.tx_flags = reinterpret_cast( - cpu_roce_get_tx_ring_flag_addr(xcvr)); - tp.rx_data = - reinterpret_cast(cpu_roce_get_rx_ring_data_addr(xcvr)); - tp.tx_data = - reinterpret_cast(cpu_roce_get_tx_ring_data_addr(xcvr)); - tp.shutdown = [xcvr, monitor] { - cpu_roce_close(xcvr); - if (monitor->joinable()) - monitor->join(); - delete monitor; - cpu_roce_destroy_transceiver(xcvr); - }; - return true; -} - -// CPU RoCE bring-up for the HSB FPGA QP-exchange method, mirroring -// cuda-quantum's hsb_bridge_cpu.cpp (the proven CPU<->FPGA precedent): the -// peer QP is a CLI input (the FPGA's fixed data-plane QP, or the emulator's), -// the transceiver is created one-shot with the peer already known -// (cpu_roce_start, no TCP rendezvous / no connect step), and this server -// publishes its own QP / RKey / Buffer Addr on stdout in the canonical bridge -// handshake format. The orchestration script scrapes those values and hands -// them to the playback tool, which alone programs the FPGA SIF over the -// Hololink control plane (DataChannel::authenticate / configure_roce) -- this -// server performs NO control-plane traffic. -// -// tx_mode=RDMA_SEND: the FPGA/emulator posts receive WQEs for the -// server->FPGA direction and RDMA-WRITEs requests into our ring, exactly as -// with hsb_bridge_cpu. -bool init_cpu_roce_hsb_fpga_transport(const ServerConfig &cfg, - TransportEndpoints &tp) { - const std::size_t frame_size = - cfg.frame_size ? cfg.frame_size : cfg.slot_size; - - std::cout << "HSB FPGA QP exchange:\n" - << " Device: " << cfg.device << "\n" - << " Peer IP: " << cfg.peer_ip << "\n" - << " Remote QP: 0x" << std::hex << cfg.remote_qp << std::dec - << "\n" - << " Slots: " << cfg.num_slots << "\n" - << " Slot size: " << cfg.slot_size << " bytes\n" - << " Frame size: " << frame_size << " bytes" << std::endl; - - cpu_roce_transceiver_t xcvr = cpu_roce_create_transceiver( - cfg.device.c_str(), /*ib_port=*/1, /*tx_ibv_qp=*/cfg.remote_qp, - frame_size, /*page_size=*/cfg.slot_size, cfg.num_slots, - cfg.peer_ip.c_str(), /*forward=*/0, /*rx_only=*/0, /*tx_only=*/0, - /*unified=*/0, CPU_ROCE_TX_MODE_RDMA_SEND, /*peer_rx_base_addr=*/0, - /*peer_rx_rkey=*/0); - if (!xcvr) { - std::cerr << "ERROR: cpu_roce transceiver create failed" << std::endl; - return false; - } - if (!cpu_roce_start(xcvr)) { - std::cerr << "ERROR: cpu_roce_start failed" << std::endl; - cpu_roce_destroy_transceiver(xcvr); - return false; - } - - auto *monitor = new std::thread([xcvr] { cpu_roce_blocking_monitor(xcvr); }); - - tp.rx_flags = reinterpret_cast( - cpu_roce_get_rx_ring_flag_addr(xcvr)); - tp.tx_flags = reinterpret_cast( - cpu_roce_get_tx_ring_flag_addr(xcvr)); - tp.rx_data = - reinterpret_cast(cpu_roce_get_rx_ring_data_addr(xcvr)); - tp.tx_data = - reinterpret_cast(cpu_roce_get_tx_ring_data_addr(xcvr)); - tp.shutdown = [xcvr, monitor] { - cpu_roce_close(xcvr); - if (monitor->joinable()) - monitor->join(); - delete monitor; - cpu_roce_destroy_transceiver(xcvr); - }; - - // Canonical bridge handshake. Format MUST match hololink_bridge_common.h - // exactly -- " KEY: VALUE", single space after the colon -- because the - // orchestration script parses it with strict regexes (same contract as - // hsb_bridge_cpu.cpp and the Hololink GPU bridges). Buffer Addr is 0 with - // an iova=0 MR registration; the playback tool handles that. - std::cout << "\n=== Bridge Ready ===" << std::endl; - std::cout << " QP Number: 0x" << std::hex << cpu_roce_get_qp_number(xcvr) - << std::dec << std::endl; - std::cout << " RKey: " << cpu_roce_get_rkey(xcvr) << std::endl; - std::cout << " Buffer Addr: 0x" << std::hex << cpu_roce_get_buffer_addr(xcvr) - << std::dec << std::endl; - std::cout.flush(); - - print_ready(/*port=*/0, - "transport=cpu_roce qp_config=hsb_fpga peer_ip=" + cfg.peer_ip); - return true; -} - -#endif // QEC_HAVE_CPU_ROCE_TRANSPORT - -} // namespace - -int main(int argc, char **argv) { - ServerConfig cfg; - if (!parse_args(argc, argv, cfg)) - return 1; - - std::signal(SIGINT, on_signal); - std::signal(SIGTERM, on_signal); - - // [1] Validate the YAML and hand its path to the decoding-server service: - // the DecodingServer (one DecodingSession worker thread per decoder) builds - // the decoder instances itself when the dispatch session is created below. - std::ifstream config_file(cfg.config_path); - if (!config_file) { - std::cerr << "ERROR: cannot open config file " << cfg.config_path - << std::endl; - return 1; - } - std::stringstream config_text; - config_text << config_file.rdbuf(); - auto decoder_config = - config::multi_decoder_config::from_yaml_str(config_text.str()); - if (decoder_config.decoders.empty()) { - std::cerr << "ERROR: no decoders parsed from " << cfg.config_path - << std::endl; - return 1; - } - ::setenv("CUDAQ_QEC_DECODER_CONFIG", cfg.config_path.c_str(), - /*overwrite=*/1); - std::cout << "Configured " << decoder_config.decoders.size() - << " decoder(s); decoder 0 type: " - << decoder_config.decoders[0].type - << "; transport: " << cfg.transport << std::endl; - - // [2a] GPU RoCE takes a completely different path: bypass the CQR - // DeviceCallService / HOST_CALL dispatcher and use DecodingServer directly. - // Must be checked before force-linking the CQR plugin (which creates a - // DecodingServer internally for the HOST_CALL path) to avoid double-init. -#ifdef QEC_HAVE_GPU_ROCE_TRANSPORT - if (cfg.transport == "gpu_roce") { - // DecodingServer(config_yaml) reads the YAML, creates GpuRoceTransceiver - // (Hololink Sensor Bridge + DOCA), loads decoder sessions, and calls - // launch_scheduler() to wire the CUDAQ device-graph scheduler to the - // Hololink ring buffers. The GPU scheduler then handles - // RX→dispatch→decode→TX autonomously; this thread just waits for signal. - // - // Construction throws when the GPU RoCE component is not linked into - // this binary (built against HSB/DOCA headers but without the - // proprietary cudevice archive) or when Hololink bring-up fails. - try { - cudaq::qec::decoding_server::DecodingServer server(cfg.config_path); - // QP/rkey/buf already printed to stdout by launch_scheduler() so the - // orchestration script can grep them before the READY line. - std::cout << "QEC_DECODING_SERVER_READY gpu_roce" << std::endl; - std::cout.flush(); - std::thread server_thread([&server] { server.run(); }); - const auto start_time_gr = std::chrono::steady_clock::now(); - while (g_shutdown.load(std::memory_order_acquire) == 0) { - const auto elapsed = - std::chrono::duration_cast( - std::chrono::steady_clock::now() - start_time_gr) - .count(); - if (elapsed > cfg.timeout_sec) - break; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - server.stop(); - server_thread.join(); - } catch (const std::exception &e) { - std::cerr << "ERROR: gpu_roce startup failed: " << e.what() << std::endl; - return 1; - } - return 0; - } -#endif - - // [2] Pull the QEC HOST_CALL function table from the decoding-server-cqr - // service plugin -- the same table the in-process host_dispatch test uses. - cudaqx_qec_realtime_device_call_service_force_link(); - auto pluginInfo = cudaqGetDeviceCallServicePluginInfo(); - if (!pluginInfo.getService) { - std::cerr << "ERROR: QEC device_call service plugin missing" << std::endl; - return 1; - } - auto *service = pluginInfo.getService(); - if (!service) { - std::cerr << "ERROR: QEC device_call service create failed" << std::endl; - return 1; - } - // The session owns the function table; keep it alive for the server's - // lifetime (the dispatcher loop below reads table.entries in place). - // Creating it also starts the DecodingServer (decoder construction + one - // worker thread per decoder) -- before the READY line below, so slow - // decoder initialization never races the first client request. - std::unique_ptr session; - try { - session = service->createDispatchSession( - cudaq::realtime::DeviceCallDispatchMode::Host); - } catch (const std::exception &e) { - std::cerr << "ERROR: decoding-server startup failed: " << e.what() - << std::endl; - return 1; - } - if (!session) { - std::cerr << "ERROR: QEC device_call service does not support host " - "dispatch" - << std::endl; - return 1; - } - const auto &table = session->dispatchTable(); - if (!table.entries || table.count == 0) { - std::cerr << "ERROR: QEC host dispatch table unavailable" << std::endl; - return 1; - } - - // [3] Bring up the selected transport (prints the READY line once the - // caller can start connecting). - TransportEndpoints tp; - if (cfg.transport == "udp") { - if (!init_udp_transport(cfg, tp)) - return 1; - } else if (cfg.transport == "cpu_roce") { -#ifdef QEC_HAVE_CPU_ROCE_TRANSPORT - if (cfg.qp_config == "hsb_fpga") { - if (!init_cpu_roce_hsb_fpga_transport(cfg, tp)) - return 1; - } else if (!init_cpu_roce_transport(cfg, tp)) - return 1; -#else - std::cerr << "ERROR: this server was built without cpu_roce transport " - "support (libcudaq-realtime-cpu-roce-transport not found)" - << std::endl; - return 1; -#endif - } else if (cfg.transport == "gpu_roce") { - // gpu_roce is handled before the CQR plugin force-link above ([2a]). - // Reaching here means QEC_HAVE_GPU_ROCE_TRANSPORT was not defined at - // build time (the server was not built with GPU RoCE support). - std::cerr << "ERROR: this server was built without gpu_roce transport " - "support (rebuild with HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR, " - "DOCA, and CUDA)" - << std::endl; - return 1; - } else { - std::cerr << "ERROR: unknown --transport=" << cfg.transport - << " (expected udp, cpu_roce, or gpu_roce)" << std::endl; - return 1; - } - - // [4] Wire the libcudaq-realtime host dispatcher to the transceiver rings, - // exactly as cpu_roce_test_daemon does. Everything from here down is - // transport-independent. - // The dispatch table is HOST_CALL-only, so the ring loop runs the inline - // HOST_CALL path with no GRAPH_LAUNCH engine (engine == nullptr). Mirrors the - // HOST_CALL-only branch in qec_realtime_session.cpp. - int dispatcher_shutdown = 0; - std::uint64_t packets_dispatched = 0; - cudaq_ringbuffer_t ringbuffer{}; - ringbuffer.rx_flags_host = tp.rx_flags; - ringbuffer.tx_flags_host = tp.tx_flags; - ringbuffer.rx_data_host = tp.rx_data; - ringbuffer.tx_data_host = tp.tx_data; - ringbuffer.rx_stride_sz = cfg.slot_size; - ringbuffer.tx_stride_sz = cfg.slot_size; - cudaq_dispatcher_config_t dispatch_config{}; - dispatch_config.num_slots = cfg.num_slots; - dispatch_config.slot_size = static_cast(cfg.slot_size); - dispatch_config.dispatch_path = CUDAQ_DISPATCH_PATH_HOST; - dispatch_config.dispatch_mode = CUDAQ_DISPATCH_HOST_CALL; - dispatch_config.skip_tx_markers = 1; - cudaq_function_table_t function_table{}; - function_table.entries = table.entries; - function_table.count = table.count; - - std::thread dispatcher_thread([&]() { - cudaq_host_ring_dispatch_loop( - &ringbuffer, &function_table, &dispatch_config, - /*engine=*/nullptr, &dispatcher_shutdown, &packets_dispatched); - }); - - // [5] Run until signalled or timed out. - const auto start_time = std::chrono::steady_clock::now(); - while (g_shutdown.load(std::memory_order_acquire) == 0) { - const auto elapsed = std::chrono::duration_cast( - std::chrono::steady_clock::now() - start_time) - .count(); - if (elapsed > cfg.timeout_sec) - break; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - // [6] Orderly shutdown. The dispatch loop polls the flag as volatile, not - // atomically; publish the store the same way qec_realtime_session.cpp does. - __atomic_store_n(&dispatcher_shutdown, 1, __ATOMIC_RELEASE); - __sync_synchronize(); - if (dispatcher_thread.joinable()) - dispatcher_thread.join(); - if (tp.shutdown) - tp.shutdown(); - // The counters are atomics and the per-shot get_corrections cadence means - // they are settled by the time a client-driven run reaches shutdown; print - // before cudaqx_qec_decoding_server_shutdown() releases the sessions. - if (const char *stats = std::getenv("QEC_DECODING_SERVER_STATS"); - stats && stats[0] != '\0') - cudaqx_qec_decoding_server_print_stats(); - // Stop the DecodingServer receive loop and join its thread before the - // process exits (a still-joinable static thread would std::terminate). - cudaqx_qec_decoding_server_shutdown(); - - std::cout << "QEC_DECODING_SERVER_DISPATCHED count=" - << cudaqx_qec_device_call_dispatch_count() << std::endl; - // Concurrency evidence for multi-logical-qubit tests: high-water mark of - // simultaneously-busy DecodingSession workers. - std::cout << "QEC_DECODING_SERVER_MAX_CONCURRENT_DECODERS count=" - << cudaqx_qec_decoding_server_max_concurrent() << std::endl; - return 0; -} diff --git a/libs/qec/tools/decoding-server/decoding_server_config.yaml b/libs/qec/tools/decoding-server/decoding_server_config.yaml deleted file mode 100644 index 21348c6f3..000000000 --- a/libs/qec/tools/decoding-server/decoding_server_config.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# ============================================================================ # -# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # -# All rights reserved. # -# # -# This source code and the accompanying materials are made available under # -# the terms of the Apache License 2.0 which accompanies this distribution. # -# ============================================================================ # - -# Decoder configuration for the two-process decoding-server test -# (test_decoding_server_udp + decoding_server). The server is -# decoder-agnostic: swap `type` / `decoder_custom_args` to run the same test -# against a different decoder. -# -# The identity-matrix H/O/D below (3-bit syndrome, one column per row) must -# stay consistent with the test kernel's block/syndrome size and its expected -# correction (syndrome bit 1 set -> correction bit 1 set). -decoders: - - id: 0 - type: pymatching - block_size: 3 - syndrome_size: 3 - H_sparse: [0, -1, 1, -1, 2, -1] - O_sparse: [0, -1, 1, -1, 2, -1] - D_sparse: [0, -1, 1, -1, 2, -1] - decoder_custom_args: - merge_strategy: smallest_weight - error_rate_vec: [0.1, 0.1, 0.1] diff --git a/libs/qec/tools/decoding-server/decoding_server_config_multi_error_lut.yaml b/libs/qec/tools/decoding-server/decoding_server_config_multi_error_lut.yaml deleted file mode 100644 index e803e50df..000000000 --- a/libs/qec/tools/decoding-server/decoding_server_config_multi_error_lut.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# ============================================================================ # -# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # -# All rights reserved. # -# # -# This source code and the accompanying materials are made available under # -# the terms of the Apache License 2.0 which accompanies this distribution. # -# ============================================================================ # - -# Decoder configuration for the two-process decoding-server test -# (test_decoding_server_udp + decoding_server): the multi_error_lut -# variant, demonstrating that swapping decoders is a config-file change only. -# -# Same identity-matrix H/O/D contract as decoding_server_config.yaml: 3-bit -# syndrome, syndrome bit 1 set -> correction bit 1 set. -decoders: - - id: 0 - type: multi_error_lut - block_size: 3 - syndrome_size: 3 - H_sparse: [0, -1, 1, -1, 2, -1] - O_sparse: [0, -1, 1, -1, 2, -1] - D_sparse: [0, -1, 1, -1, 2, -1] - decoder_custom_args: - lut_error_depth: 1 diff --git a/libs/qec/unittests/CMakeLists.txt b/libs/qec/unittests/CMakeLists.txt index 359258a51..22bba6763 100644 --- a/libs/qec/unittests/CMakeLists.txt +++ b/libs/qec/unittests/CMakeLists.txt @@ -40,27 +40,15 @@ add_dependencies(CUDAQXQECUnitTests test_decoders) gtest_discover_tests(test_decoders) add_executable(test_decoders_yaml test_decoders_yaml.cpp decoders/sample_decoder.cpp) -target_include_directories(test_decoders_yaml PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/../lib/realtime/decoding-server-cqr) target_link_libraries(test_decoders_yaml PRIVATE GTest::gtest_main - cudaq-qec - cudaq-qec-decoding-server + cudaq-qec-decoders cudaq-qec-realtime-decoding cudaq-qec-realtime-decoding-simulation cudaq::cudaq) add_dependencies(CUDAQXQECUnitTests test_decoders_yaml) gtest_discover_tests(test_decoders_yaml) -add_executable(test_decoding_server_core test_decoding_server_core.cpp) -target_link_libraries(test_decoding_server_core PRIVATE - GTest::gtest_main - cudaq-qec-decoding-server - cudaq::cudaq - CUDA::cudart) -add_dependencies(CUDAQXQECUnitTests test_decoding_server_core) -gtest_discover_tests(test_decoding_server_core) - add_executable(test_qec test_qec.cpp) target_link_libraries(test_qec PRIVATE GTest::gtest_main cudaq-qec cudaq::cudaq-stim-target) add_dependencies(CUDAQXQECUnitTests test_qec) @@ -365,8 +353,7 @@ if(CUDAQ_REALTIME_ROOT AND CMAKE_CUDA_COMPILER) PATHS ${TENSORRT_ROOT}/lib /usr/lib/x86_64-linux-gnu /usr/lib/aarch64-linux-gnu /usr/local/cuda/lib64 /usr/local/tensorrt/lib /opt/tensorrt/lib ) - if(TENSORRT_INCLUDE_DIR_FOR_PIPELINE AND TENSORRT_LIBRARY_FOR_PIPELINE AND TENSORRT_ONNX_PARSER_FOR_PIPELINE - AND TARGET cudaq-realtime-pipeline) + if(TENSORRT_INCLUDE_DIR_FOR_PIPELINE AND TENSORRT_LIBRARY_FOR_PIPELINE AND TENSORRT_ONNX_PARSER_FOR_PIPELINE) get_filename_component(_cuda_bin_pipe "${CMAKE_CUDA_COMPILER}" DIRECTORY) get_filename_component(_cuda_root_pipe "${_cuda_bin_pipe}" DIRECTORY) set(_cuda_cccl_include_pipe "${_cuda_root_pipe}/include/cccl") @@ -518,14 +505,6 @@ if(CUDAQ_REALTIME_ROOT AND CMAKE_CUDA_COMPILER) add_executable(test_realtime_qldpc_graph_decoding ${CMAKE_CURRENT_SOURCE_DIR}/realtime/qec_graph_decode_test/test_realtime_qldpc_graph_decoding.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/realtime/qec_graph_decode_test/qldpc_config_loader.cpp - ) - - set_target_properties(test_realtime_qldpc_graph_decoding PROPERTIES - CUDA_SEPARABLE_COMPILATION ON - CUDA_RESOLVE_DEVICE_SYMBOLS ON - CUDA_STANDARD 17 - LINKER_LANGUAGE CUDA ) target_include_directories(test_realtime_qldpc_graph_decoding PRIVATE @@ -533,22 +512,18 @@ if(CUDAQ_REALTIME_ROOT AND CMAKE_CUDA_COMPILER) ${CUDAQ_REALTIME_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../include ${CMAKE_SOURCE_DIR}/libs/core/include - ${CMAKE_CURRENT_SOURCE_DIR}/../lib/realtime ) target_compile_definitions(test_realtime_qldpc_graph_decoding PRIVATE TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/realtime/qec_roce_decode_test/data" - CUDAQ_REALTIME_ROOT ) target_link_libraries(test_realtime_qldpc_graph_decoding PRIVATE GTest::gtest_main CUDA::cudart ${CUDAQ_REALTIME_LIBRARY} - ${CUDAQ_REALTIME_DISPATCH_LIBRARY} $<$:${CUDAQ_REALTIME_HOST_DISPATCH_LIBRARY}> cudaq-qec-realtime-decoding - $<$:$> $<$:${_CUDAQ_LIBRARY}> $<$:${_NVQIR_LIBRARY}> ) @@ -556,7 +531,6 @@ if(CUDAQ_REALTIME_ROOT AND CMAKE_CUDA_COMPILER) target_link_options(test_realtime_qldpc_graph_decoding PRIVATE "LINKER:--allow-shlib-undefined" "LINKER:--disable-new-dtags" - "LINKER:--export-dynamic" ) # Resolve CUDAQ SDK lib dir for RPATH (libnvqir, libcudaq) @@ -569,14 +543,13 @@ if(CUDAQ_REALTIME_ROOT AND CMAKE_CUDA_COMPILER) endif() set_target_properties(test_realtime_qldpc_graph_decoding PROPERTIES - BUILD_RPATH "${CUDAQ_REALTIME_LIB_DIR};${CMAKE_BINARY_DIR}/lib;${CMAKE_BINARY_DIR}/lib/decoder-plugins;${_cudaq_sdk_lib_dir}" - INSTALL_RPATH "${CUDAQ_REALTIME_LIB_DIR};${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR};${CMAKE_BINARY_DIR}/lib/decoder-plugins;${_cudaq_sdk_lib_dir}" + BUILD_RPATH "${CUDAQ_REALTIME_LIB_DIR};${CMAKE_BINARY_DIR}/lib/decoder-plugins;${_cudaq_sdk_lib_dir}" + INSTALL_RPATH "${CUDAQ_REALTIME_LIB_DIR};${CMAKE_BINARY_DIR}/lib/decoder-plugins;${_cudaq_sdk_lib_dir}" ) add_dependencies(CUDAQXQECUnitTests test_realtime_qldpc_graph_decoding) - if((QEC_EXTERNAL_DECODERS OR DEFINED ENV{QEC_EXTERNAL_DECODERS}) - AND TARGET cudaq-qec-realtime-cudevice-proprietary) + if(QEC_EXTERNAL_DECODERS OR DEFINED ENV{QEC_EXTERNAL_DECODERS}) add_test(NAME test_realtime_qldpc_graph_decoding COMMAND test_realtime_qldpc_graph_decoding) set_tests_properties(test_realtime_qldpc_graph_decoding PROPERTIES diff --git a/libs/qec/unittests/decoders/pymatching/CMakeLists.txt b/libs/qec/unittests/decoders/pymatching/CMakeLists.txt index cc8d5a0f8..f8aa254d7 100644 --- a/libs/qec/unittests/decoders/pymatching/CMakeLists.txt +++ b/libs/qec/unittests/decoders/pymatching/CMakeLists.txt @@ -79,15 +79,6 @@ if(CUDAQ_REALTIME_INCLUDE_DIR AND TARGET cudaq-qec-realtime-decoding-server-cqr) # the server). # - decoding-server-cqr: the host-dispatch service that receives them (and # provides the force-link + self-check hooks the test asserts on). - # The cudaq runtime resolves the quantum platform at runtime via - # dlopen(NULL) + dlsym("getQuantumPlatform") (common/PluginUtils.h), so - # libcudaq-platform-default.so must already be loaded even though nothing - # references its symbols at link time. GNU's default --as-needed would drop - # it (and libcudaq-em-default.so) from DT_NEEDED, and the test then fails - # with "undefined symbol: getQuantumPlatform"; keep them. - target_link_options(test_pymatching_device_call_realtime PRIVATE - $<$:-Wl,--no-as-needed>) - target_link_libraries(test_pymatching_device_call_realtime PRIVATE GTest::gtest_main cudaq-qec-decoders diff --git a/libs/qec/unittests/decoders/pymatching/test_pymatching_device_call_realtime.cpp b/libs/qec/unittests/decoders/pymatching/test_pymatching_device_call_realtime.cpp index 43977d6ff..4c0957b43 100644 --- a/libs/qec/unittests/decoders/pymatching/test_pymatching_device_call_realtime.cpp +++ b/libs/qec/unittests/decoders/pymatching/test_pymatching_device_call_realtime.cpp @@ -61,11 +61,12 @@ config::multi_decoder_config make_config() { decoder_config.O_sparse = identity_sparse_matrix; decoder_config.D_sparse = identity_sparse_matrix; - cudaqx::heterogeneous_map pymatching_args; - pymatching_args.insert("error_rate_vec", - std::vector(kBlockSize, kUniformErrorRate)); - pymatching_args.insert("merge_strategy", "smallest_weight"); - decoder_config.decoder_custom_args = pymatching_args; + decoder_config.decoder_custom_args = config::pymatching_config(); + auto &pymatching_config = + std::get(decoder_config.decoder_custom_args); + pymatching_config.error_rate_vec = + std::vector(kBlockSize, kUniformErrorRate); + pymatching_config.merge_strategy = "smallest_weight"; config::multi_decoder_config multi_config; multi_config.decoders.push_back(decoder_config); diff --git a/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp b/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp index 150de67fb..81a2cf6c0 100644 --- a/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp +++ b/libs/qec/unittests/decoders/pymatching/test_pymatching_realtime.cpp @@ -51,8 +51,8 @@ void expect_corrections(cudaq::qec::realtime::qec_realtime_session &session, std::span expected, std::uint64_t counter, bool reset_on_read = true) { cudaq::qec::decoding::rpc_producer::enqueue_syndromes( - session, /*decoder_id=*/0, syndrome.data(), syndrome.size(), - /*tag=*/counter); + session, /*decoder_id=*/0, syndrome.data(), syndrome.size(), counter, + /*syndrome_mapping_id=*/0); std::vector corrections(expected.size(), 0xCC); cudaq::qec::decoding::rpc_producer::get_corrections( @@ -149,7 +149,8 @@ TEST(PyMatchingRealtime, RejectsOversizedSyndromeRequest) { std::vector oversized_syndrome(oversized, 0); EXPECT_THROW(cudaq::qec::decoding::rpc_producer::enqueue_syndromes( session, /*decoder_id=*/0, oversized_syndrome.data(), - oversized_syndrome.size(), /*tag=*/1), + oversized_syndrome.size(), /*counter=*/1, + /*syndrome_mapping_id=*/0), std::runtime_error); // A correctly-sized request to the same decoder is still accepted (confirms @@ -157,7 +158,7 @@ TEST(PyMatchingRealtime, RejectsOversizedSyndromeRequest) { std::vector ok_syndrome(capacity, 0); EXPECT_NO_THROW(cudaq::qec::decoding::rpc_producer::enqueue_syndromes( session, /*decoder_id=*/0, ok_syndrome.data(), ok_syndrome.size(), - /*tag=*/2)); + /*counter=*/2, /*syndrome_mapping_id=*/0)); session.finalize(); } @@ -174,10 +175,11 @@ TEST(PyMatchingRealtime, ConfiguresViaRealtimeDecoderConfig) { decoder_config.O_sparse = {0, -1, 1, -1, 2, -1}; decoder_config.D_sparse = {0, -1, 1, -1, 2, -1}; - cudaqx::heterogeneous_map pymatching_args; - pymatching_args.insert("error_rate_vec", std::vector{0.1, 0.1, 0.1}); - pymatching_args.insert("merge_strategy", "smallest_weight"); - decoder_config.decoder_custom_args = pymatching_args; + decoder_config.decoder_custom_args = config::pymatching_config(); + auto &pymatching_config = + std::get(decoder_config.decoder_custom_args); + pymatching_config.error_rate_vec = std::vector{0.1, 0.1, 0.1}; + pymatching_config.merge_strategy = "smallest_weight"; config::multi_decoder_config multi_config; multi_config.decoders.push_back(decoder_config); diff --git a/libs/qec/unittests/decoders/realtime/test_gpu_kernels.cu b/libs/qec/unittests/decoders/realtime/test_gpu_kernels.cu index f4786f085..85a99976d 100644 --- a/libs/qec/unittests/decoders/realtime/test_gpu_kernels.cu +++ b/libs/qec/unittests/decoders/realtime/test_gpu_kernels.cu @@ -144,11 +144,6 @@ std::vector load_syndromes(const std::string &path, } current_entry = SyndromeEntry{}; reading_shot = true; - } else if (line.find("ROUND_START") == 0) { - // Per-callback round marker introduced for the per-round dispatch - // demo. This loader concatenates per-shot bits and ignores round - // boundaries -- skip ROUND_START silently. - continue; } else if (reading_shot) { // Trim whitespace line.erase(0, line.find_first_not_of(" \t\n\r")); @@ -313,7 +308,6 @@ TEST_F(GpuKernelsTest, PostprocessObservableBasic) { CUDA_CHECK(cudaMalloc(&d_O_row_ptr, O_row_ptr_.size() * sizeof(uint32_t))); CUDA_CHECK(cudaMalloc(&d_O_col_idx, O_col_idx_.size() * sizeof(uint32_t))); CUDA_CHECK(cudaMalloc(&d_corrections, num_observables_)); - CUDA_CHECK(cudaMemset(d_corrections, 0, num_observables_)); // Copy data to device CUDA_CHECK(cudaMemcpy(d_soft_decisions, soft_decisions.data(), diff --git a/libs/qec/unittests/decoders/realtime/test_realtime_decoding.cu b/libs/qec/unittests/decoders/realtime/test_realtime_decoding.cu index 12bb68bc7..6286d030f 100644 --- a/libs/qec/unittests/decoders/realtime/test_realtime_decoding.cu +++ b/libs/qec/unittests/decoders/realtime/test_realtime_decoding.cu @@ -599,8 +599,7 @@ TEST_F(RealtimeDecodingTest, DispatchKernelAllShotsGraphLaunch) { cudaError_t create_err = cudaq_create_dispatch_graph_regular( rx_flags_, tx_flags_, rx_data_, tx_data_, slot_size_, slot_size_, d_function_entries_, func_count_, d_graph_io_ctx_, d_shutdown_flag_, - d_stats_, num_slots_, 1, 32, /*triggered_graph_exec=*/nullptr, - dispatch_stream, &dispatch_ctx); + d_stats_, num_slots_, 1, 32, dispatch_stream, &dispatch_ctx); if (create_err != cudaSuccess) { cudaStreamDestroy(dispatch_stream); diff --git a/libs/qec/unittests/realtime/CMakeLists.txt b/libs/qec/unittests/realtime/CMakeLists.txt index 6f69dcb63..8230421fa 100644 --- a/libs/qec/unittests/realtime/CMakeLists.txt +++ b/libs/qec/unittests/realtime/CMakeLists.txt @@ -241,68 +241,3 @@ else() message(WARNING "hololink_predecoder_bridge: skipped (missing deps). " "Requires hololink bridge libs + TensorRT + cudaq-realtime-pipeline.") endif() - -# =========================================================================== # -# Two-process decoding-server test (UDP transport) -# =========================================================================== # -# decoding_server is a decoder-agnostic decoding-server process: the -# service end of the CUDA-Q `udp` DeviceCallChannel (the loopback stand-in for -# the RoCE transport on hosts without an RDMA NIC), with its decoders selected -# entirely by a YAML config file (decoding_server_config.yaml -- currently -# pymatching; swap the decoder by editing the config, not the code). -# test_decoding_server runs the caller (simulated QPU) side and spawns the -# server as a subprocess. - -# The decoding-server server itself is built from libs/qec/tools/decoding-server -# (this test spawns it as a subprocess and receives its path + config dir via -# compile definitions). -if(CUDAQ_REALTIME_INCLUDE_DIR AND TARGET decoding_server) - include(GoogleTest) - - add_executable(test_decoding_server) - set_target_properties(test_decoding_server PROPERTIES - LINKER_LANGUAGE CXX) - target_include_directories(test_decoding_server PRIVATE - ${CUDAQ_REALTIME_INCLUDE_DIR} - ${CUDAToolkit_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/realtime/decoding-server-cqr - ) - # -DQEC_DECODING_SERVER_PATH tells the test where the server binary lives - # (the example configs are placed next to it by tools/decoding-server). It - # rides COMPILER_FLAGS because cudaqx_add_device_code compiles with nvq++, - # which does not see target_compile_definitions. - cudaqx_add_device_code(test_decoding_server - SOURCES - test_decoding_server.cpp - COMPILER_FLAGS - --target stim - -frealtime-lowering - -DQEC_DECODING_SERVER_PATH=\"$\" - ) - - target_link_directories(test_decoding_server - PRIVATE ${CUDAQ_INSTALL_DIR}/lib) - - # Caller side only: the simulation-cqr device wrappers lower the kernel's - # device_calls to the realtime frame ABI; deliberately NOT linking the - # decoding-server-cqr service, so the only decoder is the server's. - target_link_libraries(test_decoding_server PRIVATE - GTest::gtest_main - cudaq-qec - cudaq-qec-realtime-decoding - cudaq-qec-realtime-decoding-simulation-cqr - cudaq-device-call-runtime - cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime - nvqir nvqir-stim - ) - - set_target_properties(test_decoding_server PROPERTIES - BUILD_RPATH - "${CMAKE_BINARY_DIR}/lib;${CMAKE_BINARY_DIR}/lib/decoder-plugins;${CUDAQ_INSTALL_DIR}/lib" - ) - - add_dependencies(test_decoding_server decoding_server) - add_dependencies(CUDAQXQECUnitTests test_decoding_server) - gtest_discover_tests(test_decoding_server - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) -endif() diff --git a/libs/qec/unittests/realtime/app_examples/CMakeLists.txt b/libs/qec/unittests/realtime/app_examples/CMakeLists.txt index e741b06c0..f850d37bd 100644 --- a/libs/qec/unittests/realtime/app_examples/CMakeLists.txt +++ b/libs/qec/unittests/realtime/app_examples/CMakeLists.txt @@ -15,39 +15,6 @@ # build/libs/qec/unittests/realtime/app_examples/qir-*.ll #------------------------------------------------------------------------------# -# Resolve cudaq-realtime dispatch for app examples that opt into -# CUDAQ_QEC_REALTIME_MODE=inproc_rpc. -set(_app_cudaq_realtime_prefixes "") -if(CUDAQ_REALTIME_ROOT) - list(APPEND _app_cudaq_realtime_prefixes "${CUDAQ_REALTIME_ROOT}") - list(APPEND _app_cudaq_realtime_prefixes "${CUDAQ_REALTIME_ROOT}/build") -endif() -if(CUDAQ_INSTALL_PREFIX) - list(APPEND _app_cudaq_realtime_prefixes "${CUDAQ_INSTALL_PREFIX}") -endif() -find_library(_APP_CUDAQ_REALTIME_DISPATCH_LIB cudaq-realtime-dispatch - PATHS ${_app_cudaq_realtime_prefixes} PATH_SUFFIXES lib) - -function(qec_realtime_app_link_options _tgt) - target_link_options(${_tgt} PRIVATE - "LINKER:--allow-shlib-undefined" - "LINKER:--export-dynamic" - ) - if(_APP_CUDAQ_REALTIME_DISPATCH_LIB) - target_link_libraries(${_tgt} PRIVATE - ${_APP_CUDAQ_REALTIME_DISPATCH_LIB}) - endif() - if(TARGET cudaq-qec-realtime-cudevice-proprietary) - target_link_libraries(${_tgt} PRIVATE - CUDA::cudart - $) - set_property(TARGET ${_tgt} PROPERTY CUDA_SEPARABLE_COMPILATION ON) - set_property(TARGET ${_tgt} PROPERTY CUDA_RESOLVE_DEVICE_SYMBOLS ON) - set_property(TARGET ${_tgt} PROPERTY CUDA_STANDARD 17) - set_property(TARGET ${_tgt} PROPERTY LINKER_LANGUAGE CUDA) - endif() -endfunction() - #------------------------------------------------------------------------------# # Surface code 1, fully local, regular Stim simulator add_executable(surface_code-1-local) @@ -70,7 +37,6 @@ target_link_libraries(surface_code-1-local cudaq-qec-realtime-decoding-simulation cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime nvqir nvqir-stim) -qec_realtime_app_link_options(surface_code-1-local) add_test( NAME app_examples.surface_code-1-local-test-distance-3 @@ -83,29 +49,6 @@ add_test( WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) -if(_APP_CUDAQ_REALTIME_DISPATCH_LIB - AND TARGET cudaq-qec-realtime-cudevice-proprietary) - add_test( - NAME app_examples.surface_code-1-local-test-distance-3-inproc-rpc - COMMAND - bash "${CMAKE_CURRENT_SOURCE_DIR}/surface_code-1-test.sh" - ${CMAKE_CURRENT_BINARY_DIR}/surface_code-1-local - ${CMAKE_CURRENT_BINARY_DIR}/surface_code-1-local - 3 60 40 NULL 12 6 - ${CMAKE_BINARY_DIR}/lib - nv-qldpc-decoder - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - set_tests_properties( - app_examples.surface_code-1-local-test-distance-3-inproc-rpc - PROPERTIES ENVIRONMENT - "CUDAQ_QEC_REALTIME_MODE=inproc_rpc;EXTRA_CLI_ARGS=--use-relay-bp" - # The wrapper exits 77 on GPUs below compute capability 9.0 (the - # device-graph scheduler needs device-side graph launch); report that - # as SKIPPED rather than FAILED. - SKIP_RETURN_CODE 77) -endif() - # Test with sliding_window decoder add_test( NAME app_examples.surface_code-1-local-test-distance-3-sliding-window @@ -145,95 +88,6 @@ add_test( #------------------------------------------------------------------------------# -#------------------------------------------------------------------------------# -# Surface code 1, cudaq-realtime (cqr) host-dispatch wire -# -# Same application source, but the device wrappers are lowered with -# -frealtime-lowering and linked against the simulation-cqr client, so every -# cudaq::qec::decoding::* device_call crosses the cudaq-realtime ring to the -# decoding-server-cqr service (in-process here, selected via -# CUDAQ_DEVICE_CALL_CHANNEL=host_dispatch) instead of resolving to the plain -# simulation trampolines. The test additionally asserts the service's -# dispatch count is non-zero, proving the syndromes went over the wire. -#------------------------------------------------------------------------------# -if(TARGET cudaq-qec-realtime-decoding-server-cqr) - add_executable(surface_code-1-cqr) - set_target_properties(surface_code-1-cqr PROPERTIES LINKER_LANGUAGE CXX) - - cudaqx_add_device_code(surface_code-1-cqr - SOURCES - surface_code-1.cpp - COMPILER_FLAGS - --target stim - -frealtime-lowering - -DQEC_APP_CQR - ) - - target_link_directories(surface_code-1-cqr - PRIVATE ${CUDAQ_INSTALL_DIR}/lib) - - target_link_libraries(surface_code-1-cqr - PRIVATE - cudaq-qec - cudaq-qec-realtime-decoding - cudaq-qec-realtime-decoding-simulation-cqr - cudaq-qec-realtime-decoding-server-cqr - cudaq-device-call-runtime - cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime - nvqir nvqir-stim) - qec_realtime_app_link_options(surface_code-1-cqr) - - # Enable Relay-BP only when this build provides nv-qldpc support. - set(_surface_code_1_cqr_has_nv_qldpc_decoder FALSE) - if(TARGET cudaq-qec-nv-qldpc-decoder - OR QEC_EXTERNAL_DECODERS - OR DEFINED ENV{QEC_EXTERNAL_DECODERS}) - set(_surface_code_1_cqr_has_nv_qldpc_decoder TRUE) - endif() - - set(_surface_code_1_cqr_env CUDAQ_DEVICE_CALL_CHANNEL=host_dispatch) - if(TARGET cudaq-qec-realtime-cudevice-proprietary - AND _surface_code_1_cqr_has_nv_qldpc_decoder) - list(APPEND _surface_code_1_cqr_env EXTRA_CLI_ARGS=--use-relay-bp) - endif() - - add_test( - NAME app_examples.surface_code-1-cqr-test-distance-3 - COMMAND - ${CMAKE_COMMAND} -E env ${_surface_code_1_cqr_env} - bash "${CMAKE_CURRENT_SOURCE_DIR}/surface_code-1-test.sh" - ${CMAKE_CURRENT_BINARY_DIR}/surface_code-1-cqr - ${CMAKE_CURRENT_BINARY_DIR}/surface_code-1-cqr - 3 60 40 NULL 12 6 - ${CMAKE_BINARY_DIR}/lib - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - - # Two-process: same application as the udp client, with the standard - # decoding server (decoding_server, from tools/decoding-server) in a - # separate process serving the decoders configured from the app's own - # --save_dem YAML. - if(TARGET decoding_server) - add_test( - NAME app_examples.surface_code-1-cqr-two-process-test-distance-3 - COMMAND - # Non-zero threshold 60 (vs the -local test's 40): unseeded runs at - # 1000 shots land ~35-45, so 40 flakes; a broken decode path produces - # ~150+, so 60 still discriminates. - bash "${CMAKE_CURRENT_SOURCE_DIR}/surface_code-1-cqr-two-process-test.sh" - ${CMAKE_CURRENT_BINARY_DIR}/surface_code-1-cqr - 3 60 40 - $ - 12 6 - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - endif() -endif() - -# End of surface code 1, cqr host-dispatch wire -#------------------------------------------------------------------------------# - - #------------------------------------------------------------------------------# # Surface code 1, --target quantinuum --emulate add_executable(surface_code-1-quantinuum-emulate) @@ -266,7 +120,6 @@ target_link_libraries(surface_code-1-quantinuum-emulate cudaq-qec-realtime-decoding-quantinuum cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime cudaq-rest-qpu nvqir nvqir-stim) -qec_realtime_app_link_options(surface_code-1-quantinuum-emulate) add_test( NAME app_examples.surface_code-1-quantinuum-emulate-test-distance-3-in-process @@ -316,7 +169,6 @@ target_link_libraries(surface_code-2-local cudaq-qec-realtime-decoding-simulation cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime nvqir nvqir-stim) -qec_realtime_app_link_options(surface_code-2-local) add_test( NAME app_examples.surface_code-2-local-test-distance-3 @@ -378,7 +230,6 @@ target_link_libraries(surface_code-2-quantinuum-emulate cudaq-qec-realtime-decoding-quantinuum cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime cudaq-rest-qpu nvqir nvqir-stim) -qec_realtime_app_link_options(surface_code-2-quantinuum-emulate) add_test( NAME app_examples.surface_code-2-quantinuum-emulate-test-distance-3-in-process @@ -449,7 +300,6 @@ target_link_libraries(surface_code-3-local cudaq-qec-realtime-decoding-simulation cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime nvqir nvqir-stim) -qec_realtime_app_link_options(surface_code-3-local) # --- Local tests --- add_surface_code_test( @@ -501,7 +351,6 @@ target_link_libraries(surface_code-3-quantinuum-emulate cudaq-qec-realtime-decoding-quantinuum cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime cudaq-rest-qpu nvqir nvqir-stim) -qec_realtime_app_link_options(surface_code-3-quantinuum-emulate) # --- Quantinuum emulate (in-process) --- @@ -521,627 +370,4 @@ add_surface_code_test( # End of surface code 3, --target quantinuum --emulate #------------------------------------------------------------------------------# -#------------------------------------------------------------------------------# -# Surface code 4: YAML-switchable realtime decoders (pymatching / trt+pymatching -# / nv-qldpc relay-bp). The decoder is selected at realtime from the loaded YAML -# config (--yaml). -add_executable(surface_code-4-yaml) -set_target_properties(surface_code-4-yaml PROPERTIES LINKER_LANGUAGE CXX) - -cudaqx_add_device_code(surface_code-4-yaml - SOURCES - surface_code-4-yaml.cpp - COMPILER_FLAGS - --target stim -) - -target_link_directories(surface_code-4-yaml - PRIVATE ${CUDAQ_INSTALL_DIR}/lib) - -target_link_libraries(surface_code-4-yaml - PRIVATE - cudaq-qec - cudaq-qec-realtime-decoding - cudaq-qec-realtime-decoding-simulation - cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime - nvqir nvqir-stim) -qec_realtime_app_link_options(surface_code-4-yaml) - -if(TARGET cudaq-qec-pymatching) - add_dependencies(surface_code-4-yaml cudaq-qec-pymatching) -endif() -if(TARGET cudaq-qec-trt-decoder) - add_dependencies(surface_code-4-yaml cudaq-qec-trt-decoder) -endif() - -# External decoding-server form of the same application. The quantum kernels and -# YAML generation/validation stay in surface_code-4-yaml.cpp; only the device-call -# lowering and client-side simulation library differ. Decoder construction lives -# exclusively in decoding_server for this target. -if(TARGET decoding_server - AND TARGET cudaq-qec-realtime-decoding-simulation-cqr) - add_executable(surface_code-4-yaml-cqr) - set_target_properties(surface_code-4-yaml-cqr PROPERTIES LINKER_LANGUAGE CXX) - - cudaqx_add_device_code(surface_code-4-yaml-cqr - SOURCES - surface_code-4-yaml.cpp - COMPILER_FLAGS - --target stim - -frealtime-lowering - -DQEC_APP_EXTERNAL_DECODING_SERVER - ) - - target_link_directories(surface_code-4-yaml-cqr - PRIVATE ${CUDAQ_INSTALL_DIR}/lib) - - target_link_libraries(surface_code-4-yaml-cqr - PRIVATE - cudaq-qec - cudaq-qec-realtime-decoding - cudaq-qec-realtime-decoding-simulation-cqr - cudaq-device-call-runtime - cudaq::cudaq cudaq::cudaq-common cudaq-mlir-runtime - nvqir nvqir-stim) - qec_realtime_app_link_options(surface_code-4-yaml-cqr) - - set_target_properties(surface_code-4-yaml-cqr PROPERTIES - BUILD_RPATH - "${CMAKE_BINARY_DIR}/lib;${CMAKE_BINARY_DIR}/lib/decoder-plugins;${CUDAQ_INSTALL_DIR}/lib" - ) - - if(TARGET cudaq-qec-pymatching) - add_dependencies(surface_code-4-yaml-cqr cudaq-qec-pymatching) - endif() - if(TARGET cudaq-qec-trt-decoder) - add_dependencies(surface_code-4-yaml-cqr cudaq-qec-trt-decoder) - endif() - - # Test-only plugin: its realtime decode calls rendezvous at a reusable barrier, - # making three-worker overlap deterministic rather than scheduler-dependent. - add_library(cudaq-qec-concurrency-test-decoder SHARED - concurrency_test_decoder.cpp) - target_compile_features(cudaq-qec-concurrency-test-decoder PUBLIC cxx_std_20) - target_include_directories(cudaq-qec-concurrency-test-decoder - PUBLIC - ${CMAKE_SOURCE_DIR}/libs/qec/include - ${CMAKE_SOURCE_DIR}/libs/core/include) - target_link_libraries(cudaq-qec-concurrency-test-decoder - PUBLIC - cudaq::cudaq-operator - PRIVATE - cudaq::cudaq-common - cudaq-qec) - set_target_properties(cudaq-qec-concurrency-test-decoder PROPERTIES - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib/decoder-plugins - BUILD_RPATH "$ORIGIN" - ) - - add_dependencies(surface_code-4-yaml-cqr - cudaq-qec-concurrency-test-decoder) - add_dependencies(CUDAQXQECUnitTests - surface_code-4-yaml-cqr - cudaq-qec-concurrency-test-decoder) -endif() - -# Surface-code-4 YAML tests: exercise both d=3 and d=5 for each decoder entry. -# All entries run through the realtime inproc RPC mode; relay-BP is registered -# only when the private cudevice/archive and external decoder plugin are -# configured. -find_package(Python COMPONENTS Interpreter QUIET) -set(_SC4_TRY_TRT_TEST FALSE) -if(Python_Interpreter_FOUND) - execute_process( - COMMAND ${Python_EXECUTABLE} -c "import onnx" - RESULT_VARIABLE _sc4_onnx_python_result - OUTPUT_QUIET - ERROR_QUIET) - if(_sc4_onnx_python_result EQUAL 0) - set(_SC4_TRY_TRT_TEST TRUE) - endif() -endif() -# A silent skip here once masked a dead test check for a whole release cycle: -# the trt surface-code tests register only when BOTH the trt plugin target and -# python-onnx are available, so name whichever gate actually failed. -if(NOT TARGET cudaq-qec-trt-decoder) - message(WARNING "surface_code-4-yaml: trt_decoder tests NOT registered " - "(cudaq-qec-trt-decoder target not built).") -elseif(NOT _SC4_TRY_TRT_TEST) - if(Python_Interpreter_FOUND) - message(WARNING "surface_code-4-yaml: trt_decoder tests NOT registered " - "(python module 'onnx' not importable by ${Python_EXECUTABLE}); " - "pip install onnx to enable them.") - else() - message(WARNING "surface_code-4-yaml: trt_decoder tests NOT registered " - "(no python interpreter found).") - endif() -endif() - -function(add_surface_code_4_yaml_test test_suffix distance num_rounds - decoder_type num_shots onnx_path) - set(_sc4_env "CUDAQ_QEC_REALTIME_MODE=inproc_rpc") - if(Python_Interpreter_FOUND) - list(APPEND _sc4_env "PYTHON=${Python_EXECUTABLE}") - endif() - - add_test( - NAME app_examples.surface_code-4-yaml-${test_suffix} - COMMAND - bash "${CMAKE_CURRENT_SOURCE_DIR}/surface_code-4-yaml-test.sh" - ${CMAKE_CURRENT_BINARY_DIR}/surface_code-4-yaml - ${distance} ${num_rounds} - ${decoder_type} ${num_shots} "${onnx_path}" ${ARGN} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - set_tests_properties( - app_examples.surface_code-4-yaml-${test_suffix} - PROPERTIES ENVIRONMENT "${_sc4_env}") -endfunction() - -function(add_surface_code_4_yaml_external_test test_suffix distance num_rounds - decoder_type num_shots onnx_path) - set(_sc4_external_env - "QEC_DECODING_SERVER=$") - if(Python_Interpreter_FOUND) - list(APPEND _sc4_external_env "PYTHON=${Python_EXECUTABLE}") - endif() - - add_test( - NAME app_examples.surface_code-4-yaml-cqr-${test_suffix} - COMMAND - ${CMAKE_COMMAND} -E env ${_sc4_external_env} - bash "${CMAKE_CURRENT_SOURCE_DIR}/surface_code-4-yaml-test.sh" - $ - ${distance} ${num_rounds} ${decoder_type} ${num_shots} "${onnx_path}" - --num_logical 3 ${ARGN} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - set_tests_properties( - app_examples.surface_code-4-yaml-cqr-${test_suffix} - PROPERTIES TIMEOUT 300) -endfunction() - -# This example decodes ONE volume of num_rounds rounds (no sliding windows), so -# each case runs num_rounds == distance rounds. -function(add_surface_code_4_yaml_decoder_cases label decoder_type onnx_path) - add_surface_code_4_yaml_test(${label}-d3 3 3 - ${decoder_type} 200 "${onnx_path}" ${ARGN}) - add_surface_code_4_yaml_test(${label}-d5 5 5 - ${decoder_type} 200 "${onnx_path}" ${ARGN}) -endfunction() - -add_surface_code_4_yaml_decoder_cases(pymatching pymatching "") - -# TensorRT predecoder + PyMatching global decoder. The shell driver generates a -# tiny ONNX model sized for each test case: output [pre_L=0, residual], so TRT -# preserves the syndrome and PyMatching performs the correction. -if(TARGET cudaq-qec-trt-decoder AND _SC4_TRY_TRT_TEST) - add_surface_code_4_yaml_decoder_cases(trt-pymatching trt_decoder AUTO) -endif() - -# nv-qldpc relay-BP requires both the external nv-qldpc decoder plugin and the -# proprietary cudevice archive. Match the other realtime graph/inproc tests: -# register only when those dependencies are configured; do not runtime-skip. -if(_APP_CUDAQ_REALTIME_DISPATCH_LIB - AND TARGET cudaq-qec-realtime-cudevice-proprietary - AND (QEC_EXTERNAL_DECODERS OR DEFINED ENV{QEC_EXTERNAL_DECODERS})) - add_surface_code_4_yaml_decoder_cases(relay-bp nv-qldpc-decoder "" - --use-relay-bp) -endif() - -# Relaxed round rule (T=6 not a multiple of d=5) and multi-logical patches, each -# through the full generate + reload + decode driver -- so no hand-written -# generate/reload duplication, and both get the driver's residual-LER check. -add_surface_code_4_yaml_test(d5t6 5 6 pymatching 20 "") -add_surface_code_4_yaml_test(multilogical 3 3 pymatching 20 "" --num_logical 2) - -# Multi-decoder-type cases: --decoder_type lists one decoder per patch (patch i -# decodes with entry i through one realtime session). list-parse exercises the -# list machinery with no new dependencies (homogeneous pair); the trio cases -# bind three patches to heterogeneous decoder configs -- d5 exercises the -# vector result path beyond the old num_logical*d^2 < 64 packing ceiling. -add_surface_code_4_yaml_test(list-parse-d3 3 3 "pymatching,pymatching" 20 "" - --num_logical 2) -if(TARGET cudaq-qec-trt-decoder AND _SC4_TRY_TRT_TEST) - add_surface_code_4_yaml_test(trio-trt-d3 3 3 - "pymatching,trt_decoder,pymatching" 200 AUTO --num_logical 3) - add_surface_code_4_yaml_test(trio-trt-d5 5 5 - "pymatching,trt_decoder,pymatching" 200 AUTO --num_logical 3) -endif() - -if(TARGET surface_code-4-yaml-cqr) - add_surface_code_4_yaml_external_test(three-way-overlap 3 3 - "concurrency_test_decoder,concurrency_test_decoder,concurrency_test_decoder" - 2 "") - set_property(TEST app_examples.surface_code-4-yaml-cqr-three-way-overlap - APPEND PROPERTY ENVIRONMENT - "QEC_CONCURRENCY_TEST_DECODERS=3;REQUIRE_DECODER_CONCURRENCY=3;EXPECTED_BARRIER_COMPLETIONS=2;EXPECTED_DECODER_CORRECTIONS=0,2,0;EXPECTED_SERVER_DECODER_CONSTRUCTIONS=3;CHECK_MISSING_SERVER_PORT=1;SKIP_LOGICAL_ERROR_CEILING=1") - - if(TARGET cudaq-qec-trt-decoder AND _SC4_TRY_TRT_TEST) - add_surface_code_4_yaml_external_test(happy-path 3 3 - "pymatching,trt_decoder,pymatching" 200 AUTO) - set_property(TEST app_examples.surface_code-4-yaml-cqr-happy-path - APPEND PROPERTY ENVIRONMENT - "REQUIRE_SERVER_DECODE_COUNTS=1") - endif() - - # A real-workload skew case: all three patches use PyMatching, but patch 1 - # receives a much higher physical error rate and a correspondingly generated - # decoder model. This exercises independent server workers without a - # test-only delay in the decoder implementation. - add_surface_code_4_yaml_external_test(hard-patch-pymatching 7 7 - "pymatching,pymatching,pymatching" 100 "" - --p_spam_per_patch 0.001,0.20,0.001) - set_property(TEST app_examples.surface_code-4-yaml-cqr-hard-patch-pymatching - APPEND PROPERTY ENVIRONMENT - "CHECK_HARD_PATCH_MODELS=1;REQUIRE_DECODER_CONCURRENCY=2;SKIP_LOGICAL_ERROR_CEILING=1") - - # Ising bundles are generated outside this repository. Supplying a bundle at - # configure time registers an external-server test using the same identity - # TRT predecoder as the mocked happy path, but with Ising H/O/priors/D. - set(QEC_SURFACE_CODE_4_ISING_BUNDLE "" CACHE PATH - "Ising d/T/Z bundle used by the external surface-code example") - set(QEC_SURFACE_CODE_4_ISING_DISTANCE 7 CACHE STRING - "Distance encoded by QEC_SURFACE_CODE_4_ISING_BUNDLE") - set(QEC_SURFACE_CODE_4_ISING_ROUNDS 7 CACHE STRING - "Round count encoded by QEC_SURFACE_CODE_4_ISING_BUNDLE") - set(QEC_SURFACE_CODE_4_ISING_P_SPAM "0.01" CACHE STRING - "Runtime SPAM rate matching the Ising bundle priors") - if(QEC_SURFACE_CODE_4_ISING_BUNDLE - AND TARGET cudaq-qec-trt-decoder AND _SC4_TRY_TRT_TEST) - add_surface_code_4_yaml_external_test(ising-identity - ${QEC_SURFACE_CODE_4_ISING_DISTANCE} - ${QEC_SURFACE_CODE_4_ISING_ROUNDS} - "pymatching,trt_decoder,pymatching" 20 AUTO - --ising_bundle "${QEC_SURFACE_CODE_4_ISING_BUNDLE}" - --p_spam_per_patch "${QEC_SURFACE_CODE_4_ISING_P_SPAM}") - set_property(TEST app_examples.surface_code-4-yaml-cqr-ising-identity - APPEND PROPERTY ENVIRONMENT - "REQUIRE_TRT_EXECUTION=1;SKIP_LOGICAL_ERROR_CEILING=1") - endif() -endif() - -# The multi-type cases must PROVE the patches traveled through one realtime -# session: without realtime support maybe_init_realtime_session() is a no-op -# and enqueue/get/reset silently take the legacy direct-call path, so the -# tests would pass on YAML routing alone. Assert HOST-mode initialization -# whenever this build has realtime support. Gate on the compile definition -# actually applied to cudaq-qec-realtime-decoding (set when -# CUDAQ_REALTIME_INCLUDE_DIR is discovered), NOT the optional -# CUDAQ_REALTIME_ROOT search hint -- the headers can be found through -# CUDAQ_INSTALL_DIR while the hint stays empty. -get_target_property(_sc4_rt_defs cudaq-qec-realtime-decoding - COMPILE_DEFINITIONS) -if(_sc4_rt_defs MATCHES "CUDAQ_REALTIME_ROOT") - set(_sc4_hostmode_env "REQUIRE_HOST_MODE=1;CUDAQ_LOG_LEVEL=info") - set_property(TEST app_examples.surface_code-4-yaml-list-parse-d3 APPEND - PROPERTY ENVIRONMENT "${_sc4_hostmode_env}") - if(TARGET cudaq-qec-trt-decoder AND _SC4_TRY_TRT_TEST) - set_property(TEST app_examples.surface_code-4-yaml-trio-trt-d3 APPEND - PROPERTY ENVIRONMENT "${_sc4_hostmode_env}") - set_property(TEST app_examples.surface_code-4-yaml-trio-trt-d5 APPEND - PROPERTY ENVIRONMENT "${_sc4_hostmode_env}") - endif() -endif() - -# Three-way heterogeneous trio incl. relay-BP. Requires a HOST-dispatch -# (non-graph) nv-qldpc plugin via QEC_EXTERNAL_DECODERS, built against the -# CURRENT cudaq-qec headers (a stale plugin fails to load): the inproc session -# must be dispatch-homogeneous, and pymatching/trt are host-mode. A -# graph-flavor plugin (built with CUDAQ_REALTIME_ROOT) would be rejected as a -# mixed set. 1000 shots make working-vs-broken unambiguous for the BP entry -# under the predeclared 2% correctness bound. REQUIRE_HOST_MODE + -# CUDAQ_LOG_LEVEL=info make the driver assert the realtime session actually -# initialized (no vacuous pass through the legacy direct-call stub). -if((QEC_EXTERNAL_DECODERS OR DEFINED ENV{QEC_EXTERNAL_DECODERS}) - AND TARGET cudaq-qec-trt-decoder AND _SC4_TRY_TRT_TEST) - add_surface_code_4_yaml_test(trio-relay-d3 3 3 - "pymatching,trt_decoder,nv-qldpc-decoder" 1000 AUTO - --num_logical 3 --use-relay-bp) - set_property(TEST app_examples.surface_code-4-yaml-trio-relay-d3 APPEND - PROPERTY ENVIRONMENT "REQUIRE_HOST_MODE=1;CUDAQ_LOG_LEVEL=info") -endif() - -# --------------------------------------------------------------------------- # -# Negative / boundary tests: pin the validation behavior (distance/round guards, -# YAML-to-geometry binding, X-basis-bundle rejection, replay guards including -# structurally-incomplete captures, and multi-logical reload) so it cannot -# silently regress. Native ctests: PASS_REGULAR_EXPRESSION asserts the specific -# diagnostic (not merely a nonzero exit), and FIXTURES_SETUP makes a failed -# fixture fail its dependents. Self-contained -- no external Ising artifacts or -# ONNX models required. -# --------------------------------------------------------------------------- # -set(_sc4_exe ${CMAKE_CURRENT_BINARY_DIR}/surface_code-4-yaml) -set(_sc4_neg_dir ${CMAKE_CURRENT_BINARY_DIR}/negative) -file(MAKE_DIRECTORY ${_sc4_neg_dir}) -set(_sc4_neg_env "CUDAQ_QEC_REALTIME_MODE=inproc_rpc" - "CUDAQ_DEFAULT_SIMULATOR=stim") - -# A rejection case must satisfy BOTH conditions: a nonzero exit AND the expected -# diagnostic. CTest's PASS_REGULAR_EXPRESSION ignores the exit code, so a wrapper -# enforces both -- a future regression that prints the message but returns 0 -# then fails the test. The bash script is a literal bracket argument; the -# diagnostic, executable, and app args are passed as positionals so the -# diagnostic (which may contain quotes or leading dashes) needs no escaping. -function(_sc4_reject name diagnostic) - add_test(NAME app_examples.surface_code-4-yaml-negative-${name} - COMMAND bash -c [==[ -diag=$1; shift -if out=$("$@" 2>&1); then - printf '%s\n' "$out" - echo "NEGATIVE TEST FAIL: command returned 0; expected a nonzero exit" - exit 1 -fi -printf '%s\n' "$out" -printf '%s' "$out" | grep -qF -- "$diag" || { - echo "NEGATIVE TEST FAIL: missing expected diagnostic: $diag" - exit 1 -} -]==] _ "${diagnostic}" ${_sc4_exe} ${ARGN} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties(app_examples.surface_code-4-yaml-negative-${name} - PROPERTIES ENVIRONMENT "${_sc4_neg_env}") -endfunction() - -# Positive-path assertion: the command must exit 0 AND print the expected -# marker. Uses a wrapper for the same reason as _sc4_reject -- CTest's -# PASS_REGULAR_EXPRESSION ignores the exit code, and here we want both. -function(_sc4_accept name marker) - add_test(NAME app_examples.surface_code-4-yaml-${name} - COMMAND bash -c [==[ -mark=$1; shift -if ! out=$("$@" 2>&1); then - printf '%s\n' "$out" - echo "POSITIVE TEST FAIL: command returned nonzero; expected success" - exit 1 -fi -printf '%s\n' "$out" -printf '%s' "$out" | grep -qF -- "$mark" || { - echo "POSITIVE TEST FAIL: missing expected output: $mark" - exit 1 -} -]==] _ "${marker}" ${_sc4_exe} ${ARGN} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties(app_examples.surface_code-4-yaml-${name} - PROPERTIES ENVIRONMENT "${_sc4_neg_env}") -endfunction() - -# distance guard: an even or too-small distance is rejected (one guard, one -# diagnostic) before code construction; d=4 covers it. -_sc4_reject(distance-even "distance must be an odd integer" - --distance 4 --num_rounds 4 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/even.yml) - -# rounds guard: a single round has no cross-round detectors, so require >= 2. -_sc4_reject(rounds-too-few "must be >= 2" - --distance 3 --num_rounds 1 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/t1.yml) - -# replay requires --yaml. -_sc4_reject(replay-needs-yaml "load_syndrome requires --yaml" - --load_syndrome ${_sc4_neg_dir}/none.txt --distance 3 --num_rounds 3) - -# --decoder_type list guards: length must be 1 or num_logical, entries must -# name known decoders (a trailing comma is an empty entry), --use-relay-bp -# needs an nv-qldpc entry at generation, and the vector-result guards bound -# distance (numData < 64) and num_logical (correction bitmask width). -_sc4_reject(list-length-mismatch "expected 1 or num_logical" - --distance 3 --num_rounds 3 --num_logical 3 - --decoder_type pymatching,pymatching - --save_dem ${_sc4_neg_dir}/len.yml) -_sc4_reject(list-unknown-entry "--decoder_type entries must be" - --distance 3 --num_rounds 3 --decoder_type pymatching,foo - --save_dem ${_sc4_neg_dir}/unknown.yml) -_sc4_reject(list-trailing-comma "--decoder_type entries must be" - --distance 3 --num_rounds 3 --decoder_type "pymatching," - --save_dem ${_sc4_neg_dir}/trail.yml) -_sc4_reject(relay-without-nv "--use-relay-bp requires" - --distance 3 --num_rounds 3 --decoder_type pymatching --use-relay-bp - --save_dem ${_sc4_neg_dir}/relaynonv.yml) -_sc4_reject(distance-too-large "only numData < 64" - --distance 9 --num_rounds 9 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/d9.yml) -_sc4_reject(num-logical-too-large "--num_logical must be in [1, 64]" - --distance 3 --num_rounds 3 --num_logical 65 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/nl65.yml) -# Pathological value: pins that the range guard fires BEFORE the decoder-type -# list is replicated num_logical times (no num_logical-sized allocation). -_sc4_reject(num-logical-pathological "--num_logical must be in [1, 64]" - --distance 3 --num_rounds 3 --num_logical 2000000000 - --decoder_type pymatching --save_dem ${_sc4_neg_dir}/nlhuge.yml) -# Malformed numeric flags must error cleanly, not abort on an uncaught -# stoi/stod exception; > INT_MAX values take the same path. -_sc4_reject(distance-not-a-number "--distance expects an integer" - --distance abc --num_rounds 3 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/nan.yml) -_sc4_reject(num-logical-overflows-int "--num_logical expects an integer" - --distance 3 --num_rounds 3 --num_logical 5000000000 - --decoder_type pymatching --save_dem ${_sc4_neg_dir}/nlof.yml) -# Semantic ranges: lexically valid numbers outside the flag's domain must be -# rejected (a negative shot count would wrap to an enormous size_t; p_spam is -# advertised as [0, 1] and nan/inf must not pass). -_sc4_reject(num-shots-nonpositive "--num_shots must be >= 1" - --distance 3 --num_rounds 3 --num_shots 0 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/ns0.yml) -_sc4_reject(num-shots-negative "--num_shots must be >= 1" - --distance 3 --num_rounds 3 --num_shots -1 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/nsneg.yml) -_sc4_reject(p-spam-out-of-range "--p_spam must be a finite value in [0, 1]" - --distance 3 --num_rounds 3 --p_spam 1.5 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/ps15.yml) -_sc4_reject(p-spam-nan "--p_spam must be a finite value in [0, 1]" - --distance 3 --num_rounds 3 --p_spam nan --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/psnan.yml) -_sc4_reject(p-spam-per-patch-length "expected 1 or num_logical" - --distance 3 --num_rounds 3 --num_logical 3 - --p_spam_per_patch 0.01,0.20 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/psplen.yml) -_sc4_reject(p-spam-per-patch-malformed "comma-separated list of numbers" - --distance 3 --num_rounds 3 --num_logical 3 - --p_spam_per_patch 0.01,nope,0.01 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/pspbad.yml) -_sc4_reject(p-spam-per-patch-out-of-range "must be a finite value in [0, 1]" - --distance 3 --num_rounds 3 --num_logical 3 - --p_spam_per_patch 0.01,1.5,0.01 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/psprange.yml) -# p_spam = 0 is in [0, 1] (valid for a --yaml run) but cannot generate a DEM; -# generation must fail with a nonzero status, not exit 0 with no config file. -_sc4_reject(p-spam-zero-generation "--p_spam must be > 0 to generate" - --distance 3 --num_rounds 3 --p_spam 0 --decoder_type pymatching - --save_dem ${_sc4_neg_dir}/ps0.yml) - -# Dual-parse pin, plugin-free: generation instantiates no decoder, so this -# runs everywhere. Structural check (always): the BP entry carries the -# undecomposed hyperedge H -- strictly fewer columns than the matching entry's -# decomposed H. Full check (python3 available): the mixed-list BP entry's -# column multiset of (detector support, observable support, probability) is -# IDENTICAL to a directly generated undecomposed reference config at the same -# geometry. -function(_sc4_dual_parse_test distance num_rounds) - add_test(NAME app_examples.surface_code-4-yaml-dual-parse-gen-d${distance} - COMMAND bash -c [==[ -set -e -exe=$1; mixed=$2; pure=$3; d=$4; t=$5 -"$exe" --distance $d --num_rounds $t --num_logical 2 \ - --decoder_type pymatching,nv-qldpc-decoder --p_spam 0.01 --save_dem "$mixed" -"$exe" --distance $d --num_rounds $t \ - --decoder_type nv-qldpc-decoder --p_spam 0.01 --save_dem "$pure" -read -r nv match < <(awk '/- id:/{n++} /type:/{t[n]=$2} /block_size:/{b[n]=$2} - END{for(i=1;i<=n;i++){if(t[i]=="nv-qldpc-decoder")nv=b[i];else m=b[i]} - print nv, m}' "$mixed") -echo "nv block_size=$nv matching block_size=$match" -test -n "$nv" && test -n "$match" && test "$nv" -lt "$match" -if command -v python3 >/dev/null 2>&1; then -python3 - "$mixed" "$pure" << 'PY' -import re, sys - -def nv_columns(path): - txt = open(path).read() - for chunk in re.split(r"(?m)^\s*- id:", txt)[1:]: - typ = re.search(r"type:\s*(\S+)", chunk).group(1) - if typ != "nv-qldpc-decoder": - continue - block_size = int(re.search(r"block_size:\s*(\d+)", chunk).group(1)) - - def sparse_rows(name): - m = re.search(name + r":\s*\[([^\]]*)\]", chunk, re.S) - rows, cur = [], [] - for tok in m.group(1).replace("\n", " ").split(","): - tok = tok.strip() - if not tok: - continue - v = int(tok) - if v == -1: - rows.append(cur) - cur = [] - else: - cur.append(v) - return rows - - def col_map(rows): - cols = {} - for r, row in enumerate(rows): - for c in row: - cols.setdefault(c, []).append(r) - return cols - - hcol = col_map(sparse_rows("H_sparse")) - ocol = col_map(sparse_rows("O_sparse")) - pm = re.search(r"error_rate_vec:\s*\[([^\]]*)\]", chunk, re.S) - probs = [round(float(t), 12) - for t in pm.group(1).replace("\n", " ").split(",") - if t.strip()] - if len(probs) != block_size: - raise SystemExit("prob count %d != block_size %d in %s" % - (len(probs), block_size, path)) - return sorted((tuple(sorted(hcol.get(c, []))), - tuple(sorted(ocol.get(c, []))), probs[c]) - for c in range(block_size)) - raise SystemExit("no nv-qldpc-decoder entry in " + path) - -mixed = nv_columns(sys.argv[1]) -pure = nv_columns(sys.argv[2]) -if mixed != pure: - print("FAIL: mixed-list BP entry differs from undecomposed reference " - "(%d vs %d columns)" % (len(mixed), len(pure))) - for a, b in zip(mixed, pure): - if a != b: - print("first differing column:", a, "vs", b) - break - raise SystemExit(1) -print("full column multiset (dets, obs, prob) identical: %d columns" % - len(mixed)) -PY -else - echo "python3 not found; full multiset check skipped (structural check only)" -fi -]==] _ ${_sc4_exe} ${_sc4_neg_dir}/dualparse-mixed-d${distance}.yml - ${_sc4_neg_dir}/dualparse-pure-d${distance}.yml ${distance} ${num_rounds} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties(app_examples.surface_code-4-yaml-dual-parse-gen-d${distance} - PROPERTIES ENVIRONMENT "${_sc4_neg_env}") -endfunction() -_sc4_dual_parse_test(3 3) -_sc4_dual_parse_test(5 5) - -# X-basis Ising bundle rejected (basis must be Z). Fabricate just the -# metadata.txt the loader inspects; the dummy ONNX is never opened because the -# basis check throws first. -file(WRITE ${_sc4_neg_dir}/xbundle/metadata.txt - "distance=3\nn_rounds=3\nbasis=X\ncode_rotation=XV\n") -_sc4_reject(xbasis-bundle "basis='X'" - --distance 3 --num_rounds 3 --decoder_type trt_decoder - --onnx_path ${_sc4_neg_dir}/dummy.onnx --ising_bundle ${_sc4_neg_dir}/xbundle - --save_dem ${_sc4_neg_dir}/xb.yml) - -# Geometry binding: a d5/T5 config rejected when loaded as d3/T15 (same 120 -# detectors, different measurement-buffer span). -add_test(NAME app_examples.surface_code-4-yaml-negative-d5t5-generate - COMMAND ${_sc4_exe} --distance 5 --num_rounds 5 --num_shots 20 --p_spam 0.01 - --decoder_type pymatching --save_dem ${_sc4_neg_dir}/d5t5.yml - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) -set_tests_properties(app_examples.surface_code-4-yaml-negative-d5t5-generate - PROPERTIES FIXTURES_SETUP sc4_d5t5 ENVIRONMENT "${_sc4_neg_env}") -_sc4_reject(geometry-mismatch "measurement-buffer span" - --yaml ${_sc4_neg_dir}/d5t5.yml --distance 3 --num_rounds 15 - --num_shots 20 --p_spam 0.01) -set_tests_properties(app_examples.surface_code-4-yaml-negative-geometry-mismatch - PROPERTIES FIXTURES_REQUIRED sc4_d5t5) - -# Replay structural integrity: build a valid d3 capture, then derive a truncated -# (footer dropped) and a corrections-flipped copy. Replaying a missing, -# truncated, or correction-mismatched capture must fail with a diagnostic. -add_test(NAME app_examples.surface_code-4-yaml-negative-replay-fixtures - COMMAND bash -c "set -e -'${_sc4_exe}' --distance 3 --num_rounds 3 --num_shots 5 --p_spam 0.01 --decoder_type pymatching --save_dem '${_sc4_neg_dir}/d3.yml' -'${_sc4_exe}' --yaml '${_sc4_neg_dir}/d3.yml' --distance 3 --num_rounds 3 --num_shots 5 --p_spam 0.01 --save_syndrome '${_sc4_neg_dir}/cap.txt' -sed '/^CORRECTIONS_START/,$d' '${_sc4_neg_dir}/cap.txt' > '${_sc4_neg_dir}/cap_trunc.txt' -sed '/CORRECTIONS_START/,/CORRECTIONS_END/{s/^0$/X/;s/^1$/0/;s/^X$/1/}' '${_sc4_neg_dir}/cap.txt' > '${_sc4_neg_dir}/cap_mismatch.txt'" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) -set_tests_properties(app_examples.surface_code-4-yaml-negative-replay-fixtures - PROPERTIES FIXTURES_SETUP sc4_replay ENVIRONMENT "${_sc4_neg_env}") - -_sc4_reject(replay-missing-file "Could not open syndrome file" - --yaml ${_sc4_neg_dir}/d3.yml --load_syndrome ${_sc4_neg_dir}/missing.txt - --distance 3 --num_rounds 3) -set_tests_properties(app_examples.surface_code-4-yaml-negative-replay-missing-file - PROPERTIES FIXTURES_REQUIRED sc4_replay) -_sc4_reject(replay-truncated "missing the CORRECTIONS_START/CORRECTIONS_END" - --yaml ${_sc4_neg_dir}/d3.yml --load_syndrome ${_sc4_neg_dir}/cap_trunc.txt - --distance 3 --num_rounds 3 --num_shots 5) -set_tests_properties(app_examples.surface_code-4-yaml-negative-replay-truncated - PROPERTIES FIXTURES_REQUIRED sc4_replay) -_sc4_reject(replay-correction-mismatch "replay correction mismatch" - --yaml ${_sc4_neg_dir}/d3.yml --load_syndrome ${_sc4_neg_dir}/cap_mismatch.txt - --distance 3 --num_rounds 3 --num_shots 5) -set_tests_properties(app_examples.surface_code-4-yaml-negative-replay-correction-mismatch - PROPERTIES FIXTURES_REQUIRED sc4_replay) -# Positive replay: the untampered capture must round-trip -- every replayed -# correction matches the captured run. Proves the load/replay path works on good -# input, not just that it rejects tampered captures. -_sc4_accept(replay-roundtrip "SUCCESS: All corrections match" - --yaml ${_sc4_neg_dir}/d3.yml --load_syndrome ${_sc4_neg_dir}/cap.txt - --distance 3 --num_rounds 3 --num_shots 5) -set_tests_properties(app_examples.surface_code-4-yaml-replay-roundtrip - PROPERTIES FIXTURES_REQUIRED sc4_replay) diff --git a/libs/qec/unittests/realtime/app_examples/concurrency_test_decoder.cpp b/libs/qec/unittests/realtime/app_examples/concurrency_test_decoder.cpp deleted file mode 100644 index d084a8e8a..000000000 --- a/libs/qec/unittests/realtime/app_examples/concurrency_test_decoder.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#include "cudaq/qec/decoder.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cudaq::qec { -namespace { - -class reusable_decode_barrier { -public: - explicit reusable_decode_barrier(std::size_t participants) - : participants_(participants) { - if (participants_ < 2) - throw std::invalid_argument( - "QEC_CONCURRENCY_TEST_DECODERS must be at least 2"); - } - - void arrive_and_wait() { - std::unique_lock lock(mutex_); - const auto generation = generation_; - if (++arrived_ == participants_) { - arrived_ = 0; - ++generation_; - std::cout << "QEC_CONCURRENCY_TEST_BARRIER generation=" << generation_ - << " participants=" << participants_ << std::endl; - cv_.notify_all(); - return; - } - - if (!cv_.wait_for(lock, std::chrono::seconds(30), - [&] { return generation_ != generation; })) - throw std::runtime_error( - "timed out waiting for concurrent decoder workers"); - } - -private: - const std::size_t participants_; - std::size_t arrived_ = 0; - std::size_t generation_ = 0; - std::mutex mutex_; - std::condition_variable cv_; -}; - -std::size_t barrier_participants() { - const char *value = std::getenv("QEC_CONCURRENCY_TEST_DECODERS"); - if (!value || value[0] == '\0') - throw std::runtime_error("QEC_CONCURRENCY_TEST_DECODERS is required by " - "concurrency_test_decoder"); - - std::size_t parsed = 0; - try { - const auto participants = std::stoull(value, &parsed); - if (parsed != std::string(value).size()) - throw std::invalid_argument("trailing characters"); - return participants; - } catch (const std::exception &) { - throw std::runtime_error( - "QEC_CONCURRENCY_TEST_DECODERS must be an integer >= 2"); - } -} - -reusable_decode_barrier &decode_barrier() { - static reusable_decode_barrier barrier(barrier_participants()); - return barrier; -} - -} // namespace - -/// Test-only decoder used to prove that independent decoding-server workers -/// enter decode concurrently. The factory performs one initialization decode -/// per instance, which is deliberately excluded from the barrier. Every -/// subsequent decode rendezvous with all configured instances before returning. -class concurrency_test_decoder : public decoder { -public: - concurrency_test_decoder(const sparse_binary_matrix &H, - const cudaqx::heterogeneous_map &) - : decoder(H) { - std::cout << "QEC_CONCURRENCY_TEST_DECODER_CONSTRUCTED" << std::endl; - set_result_type(decode_result_type::decode_to_obs); - } - - decoder_result decode(const std::vector &) override { - decoder_result result{true, - std::vector(get_num_observables(), 0.0)}; - - if (initialization_probe_) { - initialization_probe_ = false; - return result; - } - - decode_barrier().arrive_and_wait(); - - // Give decoder 1 a distinct correction so the application can verify that - // per-decoder responses were routed back to the corresponding patch. - if (!result.result.empty() && get_decoder_id() == 1) - result.result[0] = 1.0; - return result; - } - - CUDAQ_EXTENSION_CUSTOM_CREATOR_FUNCTION( - concurrency_test_decoder, - static std::unique_ptr create( - const decoder_init &init, const cudaqx::heterogeneous_map ¶ms) { - return make_pcm_decoder(init, params); - }) - -private: - bool initialization_probe_ = true; -}; - -CUDAQ_EXT_PT_REGISTER_TYPE(concurrency_test_decoder) - -} // namespace cudaq::qec diff --git a/libs/qec/unittests/realtime/app_examples/gen_dsparse_from_memory_circuit.py b/libs/qec/unittests/realtime/app_examples/gen_dsparse_from_memory_circuit.py deleted file mode 100644 index f01a4d049..000000000 --- a/libs/qec/unittests/realtime/app_examples/gen_dsparse_from_memory_circuit.py +++ /dev/null @@ -1,229 +0,0 @@ -# ============================================================================ # -# Copyright (c) 2024 - 2026 NVIDIA Corporation & Affiliates. # -# All rights reserved. # -# # -# This source code and the accompanying materials are made available under # -# the terms of the Apache License 2.0 which accompanies this distribution. # -# ============================================================================ # -"""Generate D_sparse.txt for the trt+Ising path of surface_code-4-yaml. - -The Ising decoding project (https://github.com/NVIDIA/Ising-Decoding) provides a -pretrained surface-code predecoder, and its generate_test_data.py exports a -bundle of H_csr.bin/O_csr.bin/priors.bin in Ising detector order. Because -surface_code-4-yaml reads the cudaqx live measurement buffer, D_sparse.txt -bridges the two: one row per Ising detector, whose entries are cudaqx live-buffer -measurement indices, so each row reproduces one cudaqx detector bit in Ising's -detector row order. The cudaqx live measurement stream then feeds Ising's -H/O/priors. - -Recipe (see surface_code-4-yaml-test.sh header for the full flow): - 1. In the Ising repo: generate_test_data.py --distance D --n-rounds T \ - --basis Z --code-rotation XV -> H_csr.bin/O_csr.bin/priors.bin bundle - 2. Run surface_code-4-yaml --save_dem ... once; it prints cnot_schedX_flat / - cnot_schedZ_flat lines. Save that stdout to a sched.txt. - 3. python gen_dsparse_from_memory_circuit.py D T Z XV sched.txt \ - /D_sparse.txt --ising-repo /path/to/ising/code - 4. Run surface_code-4-yaml --ising_bundle ... - -Geometry: cudaqx surface_code orientation XV corresponds to Ising code_rotation -"XV" (first_bulk X, rotated_type V) under an identity data- and X-ancilla -mapping; only the Z-ancillas are permuted. This script derives that Z-ancilla -permutation by matching cudaqx Z-stabilizer supports to Ising hz rows, then -translates Ising's detector->measurement map into the cudaqx buffer order. - -Usage: - gen_dsparse_from_memory_circuit.py \ - [--ising-repo PATH] - -Positional arguments: - distance Surface code distance D. - n_rounds Number of rounds T (Ising n_rounds; counts prep + final). - basis Measurement basis (Z). - code_rotation Ising code_rotation string, e.g. XV (first_bulk=X, rotated=V). - sched.txt File containing the app's printed cnot_schedX_flat / - cnot_schedZ_flat lines. - out Output D_sparse.txt path. - -Options: - --ising-repo PATH Path to the Ising repo's `code` directory (the one - containing qec/surface_code). Defaults to - /work/github/ising/code. -""" -import argparse -import sys -import types -from pathlib import Path - - -def parse_args(argv): - ap = argparse.ArgumentParser( - description="Generate D_sparse.txt for surface_code-4-yaml's trt+Ising " - "path.") - ap.add_argument("distance", type=int, help="surface code distance D") - ap.add_argument("n_rounds", type=int, help="number of rounds T") - ap.add_argument("basis", help="measurement basis (Z)") - ap.add_argument("code_rotation", help="Ising code_rotation string, e.g. XV") - ap.add_argument("sched", - help="file with the app's printed cnot_schedX/Z_flat lines") - ap.add_argument("out", help="output D_sparse.txt path") - ap.add_argument("--ising-repo", - default="/work/github/ising/code", - help="path to the Ising repo's `code` directory " - "(contains qec/surface_code); default %(default)s") - return ap.parse_args(argv) - - -def main(argv): - args = parse_args(argv) - d = args.distance - NR = args.n_rounds - BASIS = args.basis.upper() - ROT = args.code_rotation.upper() - SCHED = args.sched - OUT = args.out - ISING = args.ising_repo - - # --- cudaqx supports from the app's printed schedule --- - def parse_sched(line): - nums = [int(x) for x in line.split(":", 1)[1].replace(",", " ").split()] - sup = {} - for i in range(0, len(nums), 2): - sup.setdefault(nums[i], set()).add(nums[i + 1]) - return [frozenset(sup[s]) for s in sorted(sup)] - - sX = sZ = None - for ln in open(SCHED): - if ln.startswith("cnot_schedX_flat:"): - sX = parse_sched(ln) - elif ln.startswith("cnot_schedZ_flat:"): - sZ = parse_sched(ln) - assert sX and sZ, "sched.txt must contain cnot_schedX_flat / cnot_schedZ_flat" - nx, nz = len(sX), len(sZ) - - # --- Ising geometry + DEM (m2d) --- - p = Path(ISING) - if not (p / "qec" / "surface_code").is_dir(): - sys.exit(f"--ising-repo '{ISING}' does not contain qec/surface_code") - sys.path.insert(0, str(p)) - _sc = types.ModuleType("qec.surface_code") - _sc.__path__ = [str(p / "qec" / "surface_code")] - _sc.__package__ = "qec.surface_code" - sys.modules.setdefault("qec.surface_code", _sc) - from qec.surface_code.memory_circuit import SurfaceCode, MemoryCircuit - from qec.noise_model import NoiseModel - - sc = SurfaceCode(d, first_bulk_syndrome_type=ROT[0], rotated_type=ROT[1]) - ihz = [ - frozenset(i - for i in range(sc.hz.shape[1]) - if sc.hz[r, i] == 1) - for r in range(sc.hz.shape[0]) - ] - ihx = [ - frozenset(i - for i in range(sc.hx.shape[1]) - if sc.hx[r, i] == 1) - for r in range(sc.hx.shape[0]) - ] - - def perm(cqx, ising, lbl): - pi = [None] * len(cqx) - used = set() - for c, supp in enumerate(cqx): - m = [ - r for r, isup in enumerate(ising) - if isup == supp and r not in used - ] - assert len( - m) == 1, f"{lbl} ancilla {c} support {sorted(supp)} matches {m}" - pi[c] = m[0] - used.add(m[0]) - return pi - - pi_z = perm(sZ, ihz, "Z") # cudaqx Z-anc c -> ising Z row - pi_x = perm(sX, ihx, "X") - assert pi_x == list(range(nx)), "expected identity X-ancilla mapping" - - # inv_z: ising Z position -> cudaqx Z position - inv_z = [None] * nz - for c, q in enumerate(pi_z): - inv_z[q] = c - - # buffer translate: ising buffer index -> cudaqx buffer index - # per round r: X block [48r..], Z block [48r+nx..]; data [nanc..] - per_round = nx + nz - nanc = per_round * NR - - def translate(m): - if m >= nanc: - return m # data: identity - r = m // per_round - off = m % per_round - if off < nx: - return per_round * r + off # X identity - return per_round * r + nx + inv_z[off - nx] # Z permuted - - # --- Ising DEM detector->measurement map --- - DEFAULT = { - "p_prep_X": 0.002, - "p_prep_Z": 0.002, - "p_meas_X": 0.002, - "p_meas_Z": 0.002, - "p_idle_cnot_X": 0.001, - "p_idle_cnot_Y": 0.001, - "p_idle_cnot_Z": 0.001, - "p_idle_spam_X": 0.001996, - "p_idle_spam_Y": 0.001996, - "p_idle_spam_Z": 0.001996, - "p_cnot_IX": 0.0002, - "p_cnot_IY": 0.0002, - "p_cnot_IZ": 0.0002, - "p_cnot_XI": 0.0002, - "p_cnot_XX": 0.0002, - "p_cnot_XY": 0.0002, - "p_cnot_XZ": 0.0002, - "p_cnot_YI": 0.0002, - "p_cnot_YX": 0.0002, - "p_cnot_YY": 0.0002, - "p_cnot_YZ": 0.0002, - "p_cnot_ZI": 0.0002, - "p_cnot_ZX": 0.0002, - "p_cnot_ZY": 0.0002, - "p_cnot_ZZ": 0.0002 - } - nm = NoiseModel(**DEFAULT) - pp = float(nm.get_max_probability()) - circ = MemoryCircuit(distance=d, - idle_error=pp, - sqgate_error=pp, - tqgate_error=pp, - spam_error=(2.0 / 3.0) * pp, - n_rounds=NR, - basis=BASIS, - code_rotation=ROT, - noise_model=nm, - add_boundary_detectors=True) - circ.set_error_rates() - stim_c = circ.stim_circuit - det_rows = [] - meas = 0 - for inst in stim_c.flattened(): - if inst.name in ("M", "MR", "MX", "MZ", "MRX", "MRZ"): - meas += len(inst.targets_copy()) - elif inst.name == "DETECTOR": - det_rows.append(sorted(meas + t.value for t in inst.targets_copy())) - - # D_sparse[j] = translate(ising_m2d[j]) - flat = [] - for row in det_rows: - for m in row: - flat.append(translate(m)) - flat.append(-1) - with open(OUT, "w") as f: - f.write(" ".join(str(x) for x in flat) + "\n") - print(f"wrote {OUT}: {len(det_rows)} detectors, nmeas={meas}, " - f"Z-anc perm nontrivial={pi_z != list(range(nz))}") - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh b/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh deleted file mode 100644 index 80179eed2..000000000 --- a/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh +++ /dev/null @@ -1,163 +0,0 @@ -# ============================================================================ # -# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # -# All rights reserved. # -# # -# This source code and the accompanying materials are made available under # -# the terms of the Apache License 2.0 which accompanies this distribution. # -# ============================================================================ # - -# Two-process surface-code test: the surface_code-1-cqr application (simulated -# QPU + cudaq-realtime device_call channel) in one process, the standard -# decoding server (decoding_server) in the other. The server is configured -# from the same YAML the app's --save_dem pass produces, so the decoder setup -# crosses the process boundary as configuration, not code. -# -# The wire between the two processes is selected by QEC_DECODING_SERVER_TRANSPORT: -# udp (default) UDP loopback; runs anywhere. -# cpu_roce CPU RoCE RDMA channel; needs an RDMA device (real ConnectX -# or SoftRoCE/rdma_rxe) and the same topology env vars as -# CUDA-Q's CpuRoceChannelTester: -# CUDAQ_CPU_ROCE_TEST_CHANNEL_DEVICE / _CHANNEL_IP -# CUDAQ_CPU_ROCE_TEST_DAEMON_DEVICE / _DAEMON_IP -# (e.g. a SoftRoCE self-loop: both = rxe_cudaq0 / 10.88.0.1) -# -# Expected args: -# 1: path to surface_code-1-cqr executable -# 2: distance -# 3: number_of_non_zero_values_threshold -# 4: number_of_corrections_decoder_threshold -# 5: path to decoding_server executable -# 6: num_rounds -# 7: decoder_window -# 8: decoder_type (optional, defaults to multi_error_lut) - -set -e - -return_code=0 - -if [[ $# -lt 7 ]]; then - echo "Error: Expected at least 7 arguments (got $#)" - exit 1 -fi - -EXE_PATH=$1 -DISTANCE=$2 -number_of_non_zero_values_threshold=$3 -number_of_corrections_decoder_threshold=$4 -SERVER_PATH=$5 -NUM_ROUNDS=$6 -DECODER_WINDOW=$7 -DECODER_TYPE=${8:-multi_error_lut} - -export CUDAQ_DEFAULT_SIMULATOR=stim - -NUM_SHOTS=1000 - -timestamp=$(date +%Y-%m-%d-%H-%M-%S) -RNG_SUFFIX=$(od -An -N4 -i /dev/urandom | tr -d ' ' | sed 's/-//g') -FULL_SUFFIX=$timestamp-$RNG_SUFFIX - -CONFIG_FILE=config-2proc-${FULL_SUFFIX}.yml -SERVER_LOG=server-2proc-${FULL_SUFFIX}.log -APP_LOG=load_dem-2proc-${FULL_SUFFIX}.log - -# [1] Generate the decoder config (no realtime channel needed for this pass). -$EXE_PATH --distance $DISTANCE --num_rounds $NUM_ROUNDS --num_shots $NUM_SHOTS \ - --save_dem $CONFIG_FILE --decoder_window $DECODER_WINDOW \ - --decoder_type $DECODER_TYPE | tee save_dem-2proc-$FULL_SUFFIX.log - -# [2] Start the decoding server on an ephemeral port with that config. -# For udp the READY port is the UDP data port; for cpu_roce it is the TCP -# rendezvous port (the RDMA wire itself is negotiated via QP/rkey exchange). -TRANSPORT=${QEC_DECODING_SERVER_TRANSPORT:-udp} -SERVER_ARGS=(--config=$CONFIG_FILE --transport=$TRANSPORT --port=0 --timeout=300) -if [[ "$TRANSPORT" == "cpu_roce" ]]; then - SERVER_ARGS+=(--device=${CUDAQ_CPU_ROCE_TEST_DAEMON_DEVICE:-mlx5_0}) - SERVER_ARGS+=(--local-ip=${CUDAQ_CPU_ROCE_TEST_DAEMON_IP:-10.0.0.2}) -fi -$SERVER_PATH "${SERVER_ARGS[@]}" \ - > $SERVER_LOG 2>&1 & -SERVER_PID=$! -cleanup() { - kill -TERM $SERVER_PID 2>/dev/null || true - wait $SERVER_PID 2>/dev/null || true -} -trap cleanup EXIT - -# Wait for the READY line and parse the port. -SERVER_PORT="" -for _ in $(seq 1 100); do - SERVER_PORT=$(grep -m1 "QEC_DECODING_SERVER_READY" $SERVER_LOG 2>/dev/null \ - | sed -n 's/.*port=\([0-9]\+\).*/\1/p') - [[ -n "$SERVER_PORT" ]] && break - sleep 0.1 -done -if [[ -z "$SERVER_PORT" ]]; then - echo "Error: server did not print QEC_DECODING_SERVER_READY" - cat $SERVER_LOG - exit 1 -fi -echo "Decoding server ready on $TRANSPORT port $SERVER_PORT" - -# [3] Run the experiment; QEC_DECODING_SERVER_PORT routes every -# cudaq::qec::decoding device_call over the selected channel to the server -# (the app reads QEC_DECODING_SERVER_TRANSPORT for the channel type). -QEC_DECODING_SERVER_PORT=$SERVER_PORT \ - $EXE_PATH --distance $DISTANCE --num_shots $NUM_SHOTS \ - --load_dem $CONFIG_FILE --num_rounds $NUM_ROUNDS \ - --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE \ - |& tee $APP_LOG - -# [4] Stop the server and collect its dispatch count. -kill -TERM $SERVER_PID -wait $SERVER_PID 2>/dev/null || true -trap - EXIT - -num_non_zero_values=$(grep "Number of non-zero values measured :" $APP_LOG | awk -F': ' '{print $2}') -num_corrections_decoder=$(grep "Number of corrections decoder found:" $APP_LOG | awk -F': ' '{print $2}') -inproc_dispatch_count=$(grep "CQR service dispatch count:" $APP_LOG | awk -F': ' '{print $2}') -server_dispatch_count=$(grep "QEC_DECODING_SERVER_DISPATCHED" $SERVER_LOG | sed -n 's/.*count=\([0-9]\+\).*/\1/p') - -if ! [[ "$num_non_zero_values" =~ ^[0-9]+$ ]]; then - echo "Error: Number of non-zero values measured is not a number" - return_code=1 -fi -if ! [[ "$num_corrections_decoder" =~ ^[0-9]+$ ]]; then - echo "Error: Number of corrections decoder found is not a number" - return_code=1 -fi -if [[ "$num_non_zero_values" -gt $number_of_non_zero_values_threshold ]]; then - echo "Error: Number of non-zero values measured is greater than $number_of_non_zero_values_threshold (unexpected)" - return_code=1 -fi -if [[ "$num_corrections_decoder" -lt $number_of_corrections_decoder_threshold ]]; then - echo "Error: Number of corrections decoder found is less than $number_of_corrections_decoder_threshold (unexpected)" - return_code=1 -fi - -# Two-process self-verification: -# - the app's in-process service count must be 0 (nothing decoded locally), -# - the server's dispatch count must cover every shot's device_calls -# (>= 3 per shot: reset_decoder + enqueues + get_corrections). -if [[ "$inproc_dispatch_count" != "0" ]]; then - echo "Error: expected in-process CQR dispatch count 0 (got '$inproc_dispatch_count'); decode did not stay in the server" - return_code=1 -fi -min_server_dispatches=$((NUM_SHOTS * 3)) -if ! [[ "$server_dispatch_count" =~ ^[0-9]+$ ]] || \ - [[ "$server_dispatch_count" -lt $min_server_dispatches ]]; then - echo "Error: server dispatch count '$server_dispatch_count' is missing or below $min_server_dispatches; device_calls did not cross the $TRANSPORT wire" - cat $SERVER_LOG - return_code=1 -else - echo "Server dispatch count check passed ($server_dispatch_count dispatches over $TRANSPORT)" -fi - -echo "Two-process test completed for distance $DISTANCE with return code $return_code" - -# Clean up log/config files unless instructed to keep them. -if [[ -z "${KEEP_LOG_FILES}" ]]; then - rm -f $CONFIG_FILE $SERVER_LOG $APP_LOG save_dem-2proc-$FULL_SUFFIX.log -fi - -exit $return_code diff --git a/libs/qec/unittests/realtime/app_examples/surface_code-1-test.sh b/libs/qec/unittests/realtime/app_examples/surface_code-1-test.sh index 90b4cf836..e6e23c9f3 100644 --- a/libs/qec/unittests/realtime/app_examples/surface_code-1-test.sh +++ b/libs/qec/unittests/realtime/app_examples/surface_code-1-test.sh @@ -43,23 +43,6 @@ LIB_DIR=$9 DECODER_TYPE=${10:-multi_error_lut} SW_WINDOW_SIZE=${11:-$DECODER_WINDOW} SW_STEP_SIZE=${12:-1} -EXTRA_CLI_ARGS=${EXTRA_CLI_ARGS:-} - -# The inproc_rpc realtime path is served by the device-graph scheduler, which -# uses device-side graph launch -- compute capability 9.0+ (Hopper) only; the -# dispatch kernel's TRIGGER_GRAPH interception is compiled out below sm_90 and -# the enqueue RPC surfaces the raw sentinel as a non-zero status. Skip on -# older GPUs (e.g. A100/sm_80 CI runners), matching the skip in -# test_realtime_qldpc_graph_decoding. Exit code 77 pairs with the test's -# SKIP_RETURN_CODE property. -if [[ "${CUDAQ_QEC_REALTIME_MODE:-}" == "inproc_rpc" ]]; then - compute_cap=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1) - if [[ -n "$compute_cap" && "${compute_cap%%.*}" -lt 9 ]]; then - echo "SKIP: CUDAQ_QEC_REALTIME_MODE=inproc_rpc requires device-side graph" \ - "launch (compute capability 9.0+); found ${compute_cap}" - exit 77 - fi -fi export CUDAQ_DEFAULT_SIMULATOR=stim @@ -76,7 +59,7 @@ FULL_SUFFIX=$timestamp-$RNG_SUFFIX CONFIG_FILE=config-${FULL_SUFFIX}.yml # Generate the config file using the first executable. -$EXE_PATH1 --distance $DISTANCE --num_rounds $NUM_ROUNDS --num_shots $NUM_SHOTS --save_dem $CONFIG_FILE --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE --sw_window_size $SW_WINDOW_SIZE --sw_step_size $SW_STEP_SIZE $EXTRA_CLI_ARGS | tee save_dem-$FULL_SUFFIX.log +$EXE_PATH1 --distance $DISTANCE --num_rounds $NUM_ROUNDS --num_shots $NUM_SHOTS --save_dem $CONFIG_FILE --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE --sw_window_size $SW_WINDOW_SIZE --sw_step_size $SW_STEP_SIZE | tee save_dem-$FULL_SUFFIX.log export CUDAQ_DUMP_JIT_IR=${CUDAQ_DUMP_JIT_IR:-0} @@ -92,8 +75,8 @@ export CUDAQ_DUMP_JIT_IR=${CUDAQ_DUMP_JIT_IR:-0} # Use the config file using the second executable. -echo Running $EXE_PATH2 --distance $DISTANCE --num_shots $NUM_SHOTS --load_dem $CONFIG_FILE --num_rounds $NUM_ROUNDS --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE --sw_window_size $SW_WINDOW_SIZE --sw_step_size $SW_STEP_SIZE $EXTRA_CLI_ARGS -$EXE_PATH2 --distance $DISTANCE --num_shots $NUM_SHOTS --load_dem $CONFIG_FILE --num_rounds $NUM_ROUNDS --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE --sw_window_size $SW_WINDOW_SIZE --sw_step_size $SW_STEP_SIZE $EXTRA_CLI_ARGS |& tee load_dem-$FULL_SUFFIX.log +echo Running $EXE_PATH2 --distance $DISTANCE --num_shots $NUM_SHOTS --load_dem $CONFIG_FILE --num_rounds $NUM_ROUNDS --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE --sw_window_size $SW_WINDOW_SIZE --sw_step_size $SW_STEP_SIZE +$EXE_PATH2 --distance $DISTANCE --num_shots $NUM_SHOTS --load_dem $CONFIG_FILE --num_rounds $NUM_ROUNDS --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE --sw_window_size $SW_WINDOW_SIZE --sw_step_size $SW_STEP_SIZE |& tee load_dem-$FULL_SUFFIX.log # If CUDAQ_DUMP_JIT_IR is "1", then extract the QIR from the # load_dem-$FULL_SUFFIX.log file and place it in qir-$FULL_SUFFIX.ll. @@ -135,19 +118,6 @@ if [[ "$num_corrections_decoder" -lt $number_of_corrections_decoder_threshold ]] return_code=1 fi -# For the cqr host-dispatch variant, verify the device_calls actually crossed -# the cudaq-realtime ring to the in-process decoding server (the count is 0 if -# they silently bypassed to a direct trampoline). -if [[ "${CUDAQ_DEVICE_CALL_CHANNEL:-}" == "host_dispatch" ]]; then - cqr_dispatch_count=$(grep "CQR service dispatch count:" load_dem-$FULL_SUFFIX.log | awk -F': ' '{print $2}') - if ! [[ "$cqr_dispatch_count" =~ ^[0-9]+$ ]] || [[ "$cqr_dispatch_count" -eq 0 ]]; then - echo "Error: CQR service dispatch count is missing or zero; device_calls did not traverse host dispatch" - return_code=1 - else - echo "CQR service dispatch count check passed ($cqr_dispatch_count dispatches)" - fi -fi - echo "Test completed for distance $DISTANCE with return code $return_code" # ============================================================================ # @@ -162,7 +132,7 @@ SYNDROME_NUM_SHOTS=10 # Use fewer shots for syndrome test # Step 1: Run simulation with --save_syndrome to capture syndrome data # Use local executable for syndrome capture (works with any platform) echo "Step 1: Saving syndromes to $SYNDROME_FILE" -$EXE_PATH1 --distance $DISTANCE --num_shots $SYNDROME_NUM_SHOTS --load_dem $CONFIG_FILE --num_rounds $NUM_ROUNDS --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE --sw_window_size $SW_WINDOW_SIZE --sw_step_size $SW_STEP_SIZE $EXTRA_CLI_ARGS --save_syndrome $SYNDROME_FILE |& tee save_syndrome-$FULL_SUFFIX.log +$EXE_PATH1 --distance $DISTANCE --num_shots $SYNDROME_NUM_SHOTS --load_dem $CONFIG_FILE --num_rounds $NUM_ROUNDS --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE --sw_window_size $SW_WINDOW_SIZE --sw_step_size $SW_STEP_SIZE --save_syndrome $SYNDROME_FILE |& tee save_syndrome-$FULL_SUFFIX.log # Check that the syndrome file was created if [[ ! -f "$SYNDROME_FILE" ]]; then @@ -182,7 +152,7 @@ else # Step 2: Replay syndromes with --load_syndrome # Use local executable for replay (doesn't need quantum simulation) echo "Step 2: Replaying syndromes from $SYNDROME_FILE" - $EXE_PATH1 --distance $DISTANCE --num_shots $SYNDROME_NUM_SHOTS --load_dem $CONFIG_FILE --num_rounds $NUM_ROUNDS --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE --sw_window_size $SW_WINDOW_SIZE --sw_step_size $SW_STEP_SIZE $EXTRA_CLI_ARGS --load_syndrome $SYNDROME_FILE |& tee load_syndrome-$FULL_SUFFIX.log + $EXE_PATH1 --distance $DISTANCE --num_shots $SYNDROME_NUM_SHOTS --load_dem $CONFIG_FILE --num_rounds $NUM_ROUNDS --decoder_window $DECODER_WINDOW --decoder_type $DECODER_TYPE --sw_window_size $SW_WINDOW_SIZE --sw_step_size $SW_STEP_SIZE --load_syndrome $SYNDROME_FILE |& tee load_syndrome-$FULL_SUFFIX.log # Check for successful replay if grep -q "Replay complete" load_syndrome-$FULL_SUFFIX.log; then diff --git a/libs/qec/unittests/realtime/app_examples/surface_code-1.cpp b/libs/qec/unittests/realtime/app_examples/surface_code-1.cpp index 741ec4b31..80a6b68c0 100644 --- a/libs/qec/unittests/realtime/app_examples/surface_code-1.cpp +++ b/libs/qec/unittests/realtime/app_examples/surface_code-1.cpp @@ -19,75 +19,13 @@ #include #include #include -#include #include #include #include -#ifdef QEC_APP_CQR -// cqr build variant: this same application compiled with -frealtime-lowering -// and linked against the simulation-cqr client wrappers, so every -// cudaq::qec::decoding::* device_call crosses the cudaq-realtime wire instead -// of resolving to the in-process trampolines. The decoding is served either by -// the in-process decoding-server-cqr service -// (CUDAQ_DEVICE_CALL_CHANNEL=host_dispatch) or by a standalone -// decoding_server (QEC_DECODING_SERVER_PORT=). The wire to the -// server defaults to udp loopback; set QEC_DECODING_SERVER_TRANSPORT=cpu_roce -// to use the CPU RoCE RDMA channel instead (works over SoftRoCE/rdma_rxe; the -// RDMA topology comes from the same CUDAQ_CPU_ROCE_TEST_* env vars as CUDA-Q's -// CpuRoceChannelTester). -#include "cudaq/realtime.h" - -// In-process service self-check hook (defined in decoding-server-cqr): -// non-zero only if device_calls actually traversed the host-dispatch ring. -extern "C" std::uint64_t cudaqx_qec_device_call_dispatch_count(); - -namespace { -std::string env_or(const char *name, const std::string &fallback) { - const char *value = std::getenv(name); - return (value && *value) ? std::string(value) : fallback; -} - -void initialize_realtime_channel(const char *prog) { - std::vector args = {prog}; - if (const char *port = std::getenv("QEC_DECODING_SERVER_PORT"); - port && *port) { - const std::string transport = - env_or("QEC_DECODING_SERVER_TRANSPORT", "udp"); - if (transport == "cpu_roce") { - // The RDMA ring geometry (slots x slot-size) is part of the cpu_roce - // wire contract: the channel writes requests directly into the server's - // rings, so these must match decoding_server's --num-slots / - // --slot-size defaults (8 x 256). - args.push_back("--cudaq-device-call=cpu_roce"); - args.push_back("--cudaq-device-call-slots=8"); - args.push_back("--cudaq-device-call-slot-size=256"); - args.push_back("ib-device=" + - env_or("CUDAQ_CPU_ROCE_TEST_CHANNEL_DEVICE", "mlx5_0")); - args.push_back("local-ip=" + - env_or("CUDAQ_CPU_ROCE_TEST_CHANNEL_IP", "10.0.0.1")); - args.push_back("rendezvous-host=" + - env_or("CUDAQ_CPU_ROCE_TEST_DAEMON_IP", "10.0.0.2")); - args.push_back(std::string("rendezvous-port=") + port); - } else { - args.push_back("--cudaq-device-call=udp"); - args.push_back("udp-host=127.0.0.1"); - args.push_back(std::string("udp-port=") + port); - } - } - std::vector argv; - for (auto &arg : args) - argv.push_back(arg.data()); - argv.push_back(nullptr); - int argc = static_cast(args.size()); - cudaq::realtime::initialize(argc, argv.data()); -} -} // namespace -#endif - // Host-side decoding API (for syndrome capture) namespace cudaq::qec::decoding::host { -void _set_syndrome_capture_callback(void (*callback)(const uint8_t *, size_t)); +void set_syndrome_capture_callback(void (*callback)(const uint8_t *, size_t)); } // Global syndrome capture state for --save_syndrome option @@ -108,7 +46,7 @@ static int g_syndromes_per_shot = 0; void save_dem_to_file(const cudaq::qec::detector_error_model &dem, std::string dem_filename, uint64_t numSyndromesPerRound, uint64_t numLogical, const std::string &decoder_type, - int sw_window_size, int sw_step_size, bool use_relay_bp) { + int sw_window_size, int sw_step_size) { cudaq::qec::decoding::config::multi_decoder_config multi_config; for (uint64_t i = 0; i < numLogical; i++) { // We actually send 1 additional round in this example, so add 1. @@ -125,55 +63,59 @@ void save_dem_to_file(const cudaq::qec::detector_error_model &dem, numSyndromesPerRound, numRounds, /*include_first_round=*/false); if (decoder_type == "nv-qldpc-decoder") { - cudaqx::heterogeneous_map nv_args; + config.decoder_custom_args = + cudaq::qec::decoding::config::nv_qldpc_decoder_config(); + auto &nv_config = + std::get( + config.decoder_custom_args); // Basic settings - nv_args.insert("use_sparsity", true); - nv_args.insert("error_rate_vec", dem.error_rates); - nv_args.insert("max_iterations", 50); - - if (use_relay_bp) { - nv_args.insert("bp_method", 3); // min-sum+dmem (required for relay) - nv_args.insert("composition", 1); // Enable sequential relay - nv_args.insert("gamma0", 0.0); // Initial gamma value - nv_args.insert("clip_value", 200.0); - nv_args.insert("repeatable", true); - cudaqx::heterogeneous_map srelay_args; - srelay_args.insert("pre_iter", std::size_t{5}); - srelay_args.insert("num_sets", std::size_t{10}); - srelay_args.insert("stopping_criterion", "All"); - srelay_args.insert("stop_nconv", std::size_t{1}); - nv_args.insert("srelay_config", srelay_args); - nv_args.insert("gamma_dist", std::vector{0.1, 0.2}); - } else { - // OSD post-processor - nv_args.insert("use_osd", true); - nv_args.insert("osd_order", 60); - nv_args.insert("osd_method", 3); - } - config.decoder_custom_args = nv_args; + nv_config.use_sparsity = true; + nv_config.error_rate_vec = dem.error_rates; + nv_config.max_iterations = 50; + + // BP Relay required settings + // Uncomment for BP Relay specific cases + // nv_config.bp_method = 3; // min-sum+dmem (required for relay) + // nv_config.composition = 1; // Enable sequential relay + // nv_config.gamma0 = 0.0; // Initial gamma value + // Relay configuration + // nv_config.srelay_config = + // cudaq::qec::decoding::config::srelay_bp_config(); + // nv_config.srelay_config->pre_iter = 5; // Pre-iterations before relay + // nv_config.srelay_config->num_sets = 10; // Number of relay sets + // nv_config.srelay_config->stopping_criterion = + // "All"; // "All", "FirstConv", or + // "NConv" + // nv_config.srelay_config->stop_nconv = 1; // For NConv criterion + // Gamma distribution for relay legs + // nv_config.gamma_dist = {0.1, 0.2}; + + // OSD post-processor + nv_config.use_osd = true; + nv_config.osd_order = 60; + nv_config.osd_method = 3; } else if (decoder_type == "multi_error_lut") { // Original multi_error_lut configuration - cudaqx::heterogeneous_map lut_args; - lut_args.insert("lut_error_depth", 2); - config.decoder_custom_args = lut_args; + cudaq::qec::decoding::config::multi_error_lut_config lut_config; + lut_config.lut_error_depth = 2; + config.decoder_custom_args = lut_config; } else if (decoder_type == "sliding_window") { // Sliding window configuration - cudaqx::heterogeneous_map sw_args; - sw_args.insert("window_size", sw_window_size); - sw_args.insert("step_size", sw_step_size); - sw_args.insert("num_syndromes_per_round", numSyndromesPerRound); - sw_args.insert("straddle_start_round", false); - sw_args.insert("straddle_end_round", true); - sw_args.insert("inner_decoder_name", "multi_error_lut"); - // Required by sliding_window - sw_args.insert("error_rate_vec", dem.error_rates); + cudaq::qec::decoding::config::sliding_window_config sw_config; + sw_config.window_size = sw_window_size; + sw_config.step_size = sw_step_size; + sw_config.num_syndromes_per_round = numSyndromesPerRound; + sw_config.straddle_start_round = false; + sw_config.straddle_end_round = true; + sw_config.inner_decoder_name = "multi_error_lut"; + sw_config.error_rate_vec = dem.error_rates; // Required by sliding_window // Configure inner multi_error_lut decoder - cudaqx::heterogeneous_map inner_lut_args; - inner_lut_args.insert("lut_error_depth", 2); - sw_args.insert("inner_decoder_params", inner_lut_args); - config.decoder_custom_args = sw_args; + cudaq::qec::decoding::config::multi_error_lut_config lut_config; + lut_config.lut_error_depth = 2; + sw_config.multi_error_lut_params = lut_config; + config.decoder_custom_args = sw_config; } multi_config.decoders.push_back(config); @@ -207,10 +149,12 @@ void load_dem_from_file(const std::string &dem_filename, auto decoder_config = config.decoders[0]; if (decoder_config.type == "sliding_window") { - const auto &sw_args = decoder_config.decoder_custom_args.map(); + auto sw_config = + std::get( + decoder_config.decoder_custom_args); // Extract from top-level error_rate_vec (required for sliding_window) - if (sw_args.contains("error_rate_vec")) { - dem.error_rates = sw_args.get>("error_rate_vec"); + if (!sw_config.error_rate_vec.empty()) { + dem.error_rates = sw_config.error_rate_vec; } } @@ -492,8 +436,7 @@ void demo_circuit_host(const cudaq::qec::code &code, int distance, const std::string &decoder_type, int sw_window_size, int sw_step_size, bool save_syndrome = false, bool load_syndrome = false, - std::string syndrome_filename = "", - bool use_relay_bp = false) { + std::string syndrome_filename = "") { if (!code.contains_operation(statePrep)) throw std::runtime_error( "sample_memory_circuit_error - requested state prep kernel not found."); @@ -651,8 +594,7 @@ void demo_circuit_host(const cudaq::qec::code &code, int distance, if (save_dem) { save_dem_to_file(dem, dem_filename, numSyndromesPerRound, numLogical, - decoder_type, sw_window_size, sw_step_size, - use_relay_bp); + decoder_type, sw_window_size, sw_step_size); return; } } @@ -703,7 +645,7 @@ void demo_circuit_host(const cudaq::qec::code &code, int distance, g_syndrome_output_file.flush(); // Register capture callback with decoder library - cudaq::qec::decoding::host::_set_syndrome_capture_callback( + cudaq::qec::decoding::host::set_syndrome_capture_callback( [](const uint8_t *data, size_t len) { std::lock_guard lock(g_syndrome_file_mutex); if (!g_syndrome_output_file.is_open()) @@ -715,8 +657,6 @@ void demo_circuit_host(const cudaq::qec::code &code, int distance, g_syndrome_output_file << "SHOT_START " << shot_num << "\n"; } - g_syndrome_output_file << "ROUND_START " << g_syndrome_count << "\n"; - // Unpack syndrome data - each byte contains 8 bits (packed format) for (size_t i = 0; i < len; i++) { uint8_t byte = data[i]; @@ -786,8 +726,6 @@ void demo_circuit_host(const cudaq::qec::code &code, int distance, } else if (line.find("SHOT_START") == 0) { saved_syndromes.emplace_back(); reading_syndromes = true; - } else if (line.find("ROUND_START") == 0) { - continue; } else if (reading_syndromes) { try { int bit = std::stoi(line); @@ -929,7 +867,7 @@ void demo_circuit_host(const cudaq::qec::code &code, int distance, // Save corrections to file if syndrome capture was enabled if (save_syndrome && g_syndrome_output_file.is_open()) { // Disable callback to stop capturing - cudaq::qec::decoding::host::_set_syndrome_capture_callback(nullptr); + cudaq::qec::decoding::host::set_syndrome_capture_callback(nullptr); // Save logical corrections for each shot (for verification during replay) g_syndrome_output_file << "CORRECTIONS_START\n"; @@ -969,8 +907,6 @@ void show_help() { "replay.\n"); printf(" --load_syndrome Load and replay syndrome data from a " "file.\n"); - printf(" --use-relay-bp For --decoder_type nv-qldpc-decoder: select " - "Relay BP instead of the default BP + OSD block.\n"); printf(" --help Show this help message\n"); } @@ -994,7 +930,6 @@ int main(int argc, char **argv) { bool save_syndrome = false; bool load_syndrome = false; std::string syndrome_filename; - bool use_relay_bp = false; // Parse the command line arguments for (int i = 1; i < argc; i++) { @@ -1045,8 +980,6 @@ int main(int argc, char **argv) { load_syndrome = true; syndrome_filename = argv[i + 1]; i++; - } else if (arg == "--use-relay-bp") { - use_relay_bp = true; } else { printf("Unknown argument: %s\n", arg.c_str()); show_help(); @@ -1136,31 +1069,11 @@ int main(int argc, char **argv) { auto code = cudaq::qec::get_code( "surface_code", cudaqx::heterogeneous_map{{"distance", distance}}); -#ifdef QEC_APP_CQR - // The --save_dem pass runs with allow_device_calls=false (MSM contexts - // only), so the device_call channel is only needed when shots actually run. - if (!save_dem) - initialize_realtime_channel(argv[0]); -#endif - demo_circuit_host(*code, distance, p_spam, cudaq::qec::operation::prep0, num_shots, num_rounds, num_logical, dem_filename, save_dem, load_dem, decoder_window, decoder_type, sw_window_size, sw_step_size, save_syndrome, load_syndrome, - syndrome_filename, use_relay_bp); - -#ifdef QEC_APP_CQR - if (!save_dem) { - // With CUDAQ_DEVICE_CALL_CHANNEL=host_dispatch this proves the shots' - // device_calls crossed the ring to the in-process decoding server (it - // stays 0 if they bypassed to a trampoline, or if a udp channel routed - // them to an external server instead). - printf("CQR service dispatch count: %llu\n", - static_cast( - cudaqx_qec_device_call_dispatch_count())); - cudaq::realtime::finalize(); - } -#endif + syndrome_filename); // Ensure clean shutdown cudaq::qec::decoding::config::finalize_decoders(); diff --git a/libs/qec/unittests/realtime/app_examples/surface_code-2.cpp b/libs/qec/unittests/realtime/app_examples/surface_code-2.cpp index f00e45959..6f9f86a2b 100644 --- a/libs/qec/unittests/realtime/app_examples/surface_code-2.cpp +++ b/libs/qec/unittests/realtime/app_examples/surface_code-2.cpp @@ -45,9 +45,9 @@ void save_dem_to_file(const cudaq::qec::detector_error_model &dem, config.O_sparse = cudaq::qec::pcm_to_sparse_vec(dem.observables_flips_matrix); config.D_sparse = std::vector(det_mat); - cudaqx::heterogeneous_map lut_args; - lut_args.insert("lut_error_depth", 2); - config.decoder_custom_args = lut_args; + cudaq::qec::decoding::config::multi_error_lut_config lut_config; + lut_config.lut_error_depth = 2; + config.decoder_custom_args = lut_config; multi_config.decoders.push_back(config); } std::string config_str = multi_config.to_yaml_str(200); @@ -76,6 +76,9 @@ void load_dem_from_file(const std::string &dem_filename, exit(1); } auto decoder_config = config.decoders[0]; + auto multi_error_lut_config = + std::get( + decoder_config.decoder_custom_args); dem.detector_error_matrix = cudaq::qec::pcm_from_sparse_vec( decoder_config.H_sparse, decoder_config.syndrome_size, decoder_config.block_size); diff --git a/libs/qec/unittests/realtime/app_examples/surface_code-3.cpp b/libs/qec/unittests/realtime/app_examples/surface_code-3.cpp index ed0327f62..241fec967 100644 --- a/libs/qec/unittests/realtime/app_examples/surface_code-3.cpp +++ b/libs/qec/unittests/realtime/app_examples/surface_code-3.cpp @@ -43,9 +43,9 @@ create_decoder_config(uint64_t id, const cudaq::qec::detector_error_model &dem, config.H_sparse = cudaq::qec::pcm_to_sparse_vec(dem.detector_error_matrix); config.O_sparse = cudaq::qec::pcm_to_sparse_vec(dem.observables_flips_matrix); config.D_sparse = det_mat; - cudaqx::heterogeneous_map lut_args; - lut_args.insert("lut_error_depth", 2); - config.decoder_custom_args = lut_args; + cudaq::qec::decoding::config::multi_error_lut_config lut_config; + lut_config.lut_error_depth = 2; + config.decoder_custom_args = lut_config; return config; } @@ -102,6 +102,13 @@ void load_dem_from_file(const std::string &dem_filename, const auto &decoder_config_z = config.decoders[2 * i]; const auto &decoder_config_x = config.decoders[2 * i + 1]; + auto multi_error_lut_config_z = + std::get( + decoder_config_z.decoder_custom_args); + auto multi_error_lut_config_x = + std::get( + decoder_config_x.decoder_custom_args); + // Z stab decoder dem_z[i].detector_error_matrix = cudaq::qec::pcm_from_sparse_vec( decoder_config_z.H_sparse, decoder_config_z.syndrome_size, diff --git a/libs/qec/unittests/realtime/app_examples/surface_code-4-yaml-test.sh b/libs/qec/unittests/realtime/app_examples/surface_code-4-yaml-test.sh deleted file mode 100755 index f8c453093..000000000 --- a/libs/qec/unittests/realtime/app_examples/surface_code-4-yaml-test.sh +++ /dev/null @@ -1,549 +0,0 @@ -# ============================================================================ # -# Copyright (c) 2024 - 2026 NVIDIA Corporation & Affiliates. # -# All rights reserved. # -# # -# This source code and the accompanying materials are made available under # -# the terms of the Apache License 2.0 which accompanies this distribution. # -# ============================================================================ # - -# Driver for the surface_code-4-yaml realtime example. It exercises the two -# phases of the app: -# Phase 1 (generation): --save_dem --decoder_type writes a YAML -# decoder config. -# Phase 2 (realtime): --yaml loads that config and decodes; the -# decoder is read FROM the file (so --decoder_type is -# NOT passed here -- the app rejects --yaml + -# --decoder_type). -# -# The example decodes ONE volume of num_rounds rounds (no sliding windows). -# -# The driver is decoder-agnostic. For trt_decoder, pass an ONNX path as arg 6 -# or pass AUTO to generate a small [pre_L=0, residual=identity] model sized for -# the requested distance/num_rounds. Additional app args (for example -# --use-relay-bp) may follow arg 6. -# -# trt+Ising external-bundle path (NOT exercised by the AUTO ctest, which uses an -# identity predecoder): to run the example against an Ising d/T/Z predecoder, you -# need the predecoder bundle (H_csr.bin/O_csr.bin/priors.bin/metadata.txt + -# D_sparse.txt) and the ONNX model, neither of which ships in this repository. -# Generate them locally from the Ising decoding project -# (https://github.com/NVIDIA/Ising-Decoding): -# (i) bundle matrices (writes H_csr.bin/O_csr.bin/priors.bin/metadata.txt -# into ): -# python generate_test_data.py --distance D --n-rounds T --basis Z \ -# --code-rotation XV --output-dir -# (ii) D_sparse.txt aligning Ising detectors to the cudaqx live buffer (run -# the app once with --save_dem to print cnot_schedX/Z, then translate): -# surface_code-4-yaml --save_dem cfg.yml --decoder_type pymatching \ -# --distance D --num_rounds T > sched.txt -# python gen_dsparse_from_memory_circuit.py D T Z XV sched.txt \ -# /D_sparse.txt --ising-repo /path/to/ising/code -# (iii) export the ONNX predecoder predecoder_memory_dD_TT_Z.onnx. -# (iv) run the example (pass to --ising_bundle): -# surface_code-4-yaml --save_dem cfg.yml --decoder_type trt_decoder \ -# --onnx_path predecoder_memory_dD_TT_Z.onnx \ -# --ising_bundle --distance D --num_rounds T ... -# surface_code-4-yaml --yaml cfg.yml --distance D --num_rounds T ... - -set -euo pipefail - -# Expected args: -# $1 exe Path to the surface_code-4-yaml executable -# $2 distance Surface code distance (D) -# $3 num_rounds Number of measurement rounds (R >= 1; R < D is decodable -# but not fault-tolerant -- no multiple-of-D constraint) -# $4 decoder_type Decoder(s) to generate: a single type, or a comma list -# with one entry per patch (pass --num_logical N in the -# extra args). Optional, defaults to pymatching. -# $5 num_shots Number of shots (optional, defaults to 200) -# $6 onnx_path ONNX path for trt_decoder, or AUTO to generate one -# $7... extra args Extra app args to pass to generation/realtime phases - -if [[ $# -lt 3 ]]; then - echo "Error: Expected at least 3 arguments (got $#)" - echo "Usage: $0 [decoder_type=pymatching] [num_shots=200]" - exit 1 -fi - -EXE=$1 -DISTANCE=$2 -NUM_ROUNDS=$3 -DECODER_TYPE=${4:-pymatching} -NUM_SHOTS=${5:-200} -ONNX_PATH=${6:-} -EXTRA_APP_ARGS=() -if [[ $# -ge 7 ]]; then - EXTRA_APP_ARGS=("${@:7}") -fi - -export CUDAQ_DEFAULT_SIMULATOR=stim -if [[ -n "${QEC_DECODING_SERVER:-}" ]]; then - export CUDAQ_QEC_REALTIME_MODE=external_server -else - export CUDAQ_QEC_REALTIME_MODE=${CUDAQ_QEC_REALTIME_MODE:-inproc_rpc} -fi - -P_SPAM=0.01 - -# Residual logical-error ceiling: a PREDECLARED correctness bound, set at -# half the d3 UNCORRECTED logical-flip rate (~4% at p_spam=0.01; ~15% at -# d5/T6). A decoder that loads but never corrects reliably exceeds it; any -# working decoder sits well below it. This is a wiring/correctness check, not -# a performance target -- it must never be tightened toward a particular -# decoder's measured rate. Floored at 1 so small shot counts do not truncate -# the ceiling to 0. -MAX_NON_ZERO=$((NUM_SHOTS / 50)) -if [[ $MAX_NON_ZERO -lt 1 ]]; then MAX_NON_ZERO=1; fi - -# Multi-type mode: a comma list binds one decoder type per patch and the -# per-decoder ceilings below replace the aggregate ceiling (which is -# calibrated for ONE patch and would silently tighten N-fold). The same -# predeclared bound applies per decoder; cases must run enough shots that -# working-vs-broken is unambiguous for every entry (>= 1000 for BP-family -# entries, whose working residual sits closest to the bound). -IFS=',' read -r -a DECODER_TYPES <<< "$DECODER_TYPE" -MULTI_TYPE=0 -if [[ ${#DECODER_TYPES[@]} -gt 1 ]]; then MULTI_TYPE=1; fi - -# Create an isolated working directory and (by default) clean it up on exit. -WORKDIR=$(mktemp -d) -SERVER_PID="" -SERVER_LOG=$WORKDIR/server.log -MISSING_PORT_LOG=$WORKDIR/missing-port.log - -stop_server() { - if [[ -n "$SERVER_PID" ]]; then - kill -TERM "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - SERVER_PID="" - fi -} - -cleanup() { - stop_server - if [[ -z "${KEEP_LOG_FILES:-}" ]]; then - rm -rf "$WORKDIR" - else - echo "KEEP_LOG_FILES set; leaving work dir: $WORKDIR" - fi -} -trap cleanup EXIT - -CONFIG_FILE=$WORKDIR/config.yml -REALTIME_LOG=$WORKDIR/realtime.log - -if [[ ",$DECODER_TYPE," == *",trt_decoder,"* && "$ONNX_PATH" == "AUTO" ]]; then - ONNX_PATH=$WORKDIR/trt_identity_predecoder.onnx - SYNDROME_SIZE=$(((DISTANCE * DISTANCE - 1) * NUM_ROUNDS)) - PYTHON_BIN=${PYTHON:-python3} - "$PYTHON_BIN" - "$ONNX_PATH" "$SYNDROME_SIZE" <<'PY' -import sys - -import onnx -from onnx import TensorProto, helper - -output_path = sys.argv[1] -syndrome_size = int(sys.argv[2]) - -input_info = helper.make_tensor_value_info( - "input", TensorProto.FLOAT, [1, syndrome_size]) -output_info = helper.make_tensor_value_info( - "output", TensorProto.FLOAT, [1, syndrome_size + 1]) -zero = helper.make_node( - "Constant", - [], - ["pre_l"], - value=helper.make_tensor("zero", TensorProto.FLOAT, [1, 1], [0.0]), -) -concat = helper.make_node("Concat", ["pre_l", "input"], ["output"], axis=1) -graph = helper.make_graph( - [zero, concat], "trt_identity_predecoder", [input_info], [output_info]) -model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 19)]) -# IR 9 is sufficient for opset 19 and remains readable by the ONNX checker in -# the CUDA-QX development image. -model.ir_version = 9 -onnx.checker.check_model(model) -onnx.save(model, output_path) -PY -fi - -echo "==============================================================" -echo "surface_code-4-yaml test" -echo " exe = $EXE" -echo " distance = $DISTANCE" -echo " num_rounds = $NUM_ROUNDS" -echo " decoder_type = $DECODER_TYPE" -echo " num_shots = $NUM_SHOTS" -echo " realtime mode = $CUDAQ_QEC_REALTIME_MODE" -if [[ -n "$ONNX_PATH" ]]; then - echo " onnx_path = $ONNX_PATH" -fi -if [[ ${#EXTRA_APP_ARGS[@]} -gt 0 ]]; then - echo " extra args = ${EXTRA_APP_ARGS[*]}" -fi -echo " max non-zero = $MAX_NON_ZERO" -echo "==============================================================" - -return_code=0 - -# -------------------------------------------------------------------------- # -# Phase 1: generation -- write the YAML decoder config. -# -------------------------------------------------------------------------- # -echo "" -echo "=== Phase 1: generate config (--save_dem, --decoder_type $DECODER_TYPE) ===" -GEN_ARGS=( - --distance "$DISTANCE" \ - --num_rounds "$NUM_ROUNDS" \ - --num_shots "$NUM_SHOTS" \ - --p_spam "$P_SPAM" \ - --decoder_type "$DECODER_TYPE" \ - --save_dem "$CONFIG_FILE" -) -if [[ -n "$ONNX_PATH" ]]; then - GEN_ARGS+=(--onnx_path "$ONNX_PATH") -fi -if [[ ${#EXTRA_APP_ARGS[@]} -gt 0 ]]; then - GEN_ARGS+=("${EXTRA_APP_ARGS[@]}") -fi -"$EXE" "${GEN_ARGS[@]}" - -# Assert the config file was created and is non-empty. -if [[ ! -s "$CONFIG_FILE" ]]; then - echo "FAIL: config file '$CONFIG_FILE' was not created or is empty" - exit 1 -fi -echo "Config file generated: $CONFIG_FILE ($(stat -c %s "$CONFIG_FILE") bytes)" - -# Dual-parse structural proof (mixed BP + matching lists): BP entries carry -# the undecomposed hyperedge H, so their block_size must be strictly LESS -# than the matching entries' decomposed block_size. Deterministic guard that -# the per-family factorization actually happened. -if [[ "$MULTI_TYPE" -eq 1 && ",$DECODER_TYPE," == *",nv-qldpc-decoder,"* ]] \ - && [[ ",$DECODER_TYPE," == *",pymatching,"* || ",$DECODER_TYPE," == *",trt_decoder,"* ]]; then - nv_bs="" - match_bs="" - while read -r typ bs; do - if [[ "$typ" == "nv-qldpc-decoder" ]]; then nv_bs=$bs; else match_bs=$bs; fi - done < <(awk '/- id:/{n++} /type:/{t[n]=$2} /block_size:/{b[n]=$2} END{for(i=1;i<=n;i++) print t[i], b[i]}' "$CONFIG_FILE") - if [[ -z "$nv_bs" || -z "$match_bs" ]] || [[ ! "$nv_bs" -lt "$match_bs" ]]; then - echo "FAIL: dual-parse structural check: nv-qldpc block_size ('$nv_bs') must be < matching block_size ('$match_bs')" - exit 1 - fi - echo "Dual-parse structural check: nv block_size $nv_bs < matching $match_bs -- OK" -fi - -# The hard-patch experiment must carry distinct decoder priors, not merely -# inject different runtime noise into three identically configured decoders. -if [[ -n "${CHECK_HARD_PATCH_MODELS:-}" ]]; then - PYTHON_BIN=${PYTHON:-python3} - "$PYTHON_BIN" - "$CONFIG_FILE" <<'PY' -import re -import sys - -text = open(sys.argv[1], encoding="utf-8").read() -blocks = re.findall(r"(?ms)^ - id:\s+(-?\d+)\n(.*?)(?=^ - id:|\Z)", text) -priors = {} -for decoder_id, block in blocks: - match = re.search(r"error_rate_vec:\s*\[([^\]]*)\]", block, re.S) - if match: - priors[int(decoder_id)] = tuple( - round(float(value.strip()), 12) - for value in match.group(1).split(",") if value.strip()) - -if set(priors) != {0, 1, 2}: - raise SystemExit(f"expected priors for decoder ids 0,1,2; got {sorted(priors)}") -if priors[0] != priors[2]: - raise SystemExit("easy-patch decoder priors differ") -if priors[1] == priors[0]: - raise SystemExit("hard-patch decoder priors are identical to easy patches") -print("Hard-patch model check: decoder 1 priors differ; decoders 0 and 2 match -- OK") -PY -fi - -if [[ -n "${CHECK_MISSING_SERVER_PORT:-}" ]]; then - MISSING_PORT_ARGS=( - --distance "$DISTANCE" - --num_rounds "$NUM_ROUNDS" - --num_shots 1 - --p_spam "$P_SPAM" - --yaml "$CONFIG_FILE" - ) - if [[ ${#EXTRA_APP_ARGS[@]} -gt 0 ]]; then - MISSING_PORT_ARGS+=("${EXTRA_APP_ARGS[@]}") - fi - - set +e - env -u QEC_DECODING_SERVER_PORT \ - "$EXE" "${MISSING_PORT_ARGS[@]}" >"$MISSING_PORT_LOG" 2>&1 - missing_port_status=$? - set -e - - if [[ "$missing_port_status" -ne 1 ]] || ! grep -Fq \ - "Error: QEC_DECODING_SERVER_PORT is required for external decoding" \ - "$MISSING_PORT_LOG"; then - echo "FAIL: missing server port did not produce a clean configuration error" - cat "$MISSING_PORT_LOG" - exit 1 - fi - echo "Missing external-server port rejected cleanly -- OK" -fi - -# An external-server test uses the generated YAML as the server's authoritative -# decoder configuration. The application still reloads and validates that YAML, -# but its CQR build deliberately does not construct local decoders. -SERVER_PORT="" -if [[ -n "${QEC_DECODING_SERVER:-}" ]]; then - if [[ -n "${REQUIRE_SERVER_DECODE_COUNTS:-}" ]]; then - QEC_DECODING_SERVER_STATS=1 \ - "$QEC_DECODING_SERVER" --config="$CONFIG_FILE" --transport=udp --port=0 \ - --timeout=300 >"$SERVER_LOG" 2>&1 & - else - "$QEC_DECODING_SERVER" --config="$CONFIG_FILE" --transport=udp --port=0 \ - --timeout=300 >"$SERVER_LOG" 2>&1 & - fi - SERVER_PID=$! - - for _ in $(seq 1 1200); do - SERVER_PORT=$(grep -m1 "QEC_DECODING_SERVER_READY" "$SERVER_LOG" \ - 2>/dev/null | sed -n 's/.*port=\([0-9]\+\).*/\1/p' || true) - [[ -n "$SERVER_PORT" ]] && break - if ! kill -0 "$SERVER_PID" 2>/dev/null; then - break - fi - sleep 0.1 - done - - if [[ -z "$SERVER_PORT" ]]; then - echo "FAIL: decoding server did not become ready" - cat "$SERVER_LOG" - exit 1 - fi - echo "External decoding server ready on UDP port $SERVER_PORT" -fi - -# -------------------------------------------------------------------------- # -# Phase 2: realtime -- load the YAML config and decode. -# The decoder is read from the file, so do NOT pass --decoder_type here. -# -------------------------------------------------------------------------- # -echo "" -echo "=== Phase 2: realtime decode (--yaml $CONFIG_FILE) ===" -# Use a pipefail-safe tee so a crash in the app still surfaces a non-zero status -# while we keep the full log for assertions below. -set +e -REALTIME_ARGS=( - --distance "$DISTANCE" \ - --num_rounds "$NUM_ROUNDS" \ - --num_shots "$NUM_SHOTS" \ - --p_spam "$P_SPAM" \ - --yaml "$CONFIG_FILE" -) -if [[ ${#EXTRA_APP_ARGS[@]} -gt 0 ]]; then - REALTIME_ARGS+=("${EXTRA_APP_ARGS[@]}") -fi -if [[ -n "$SERVER_PORT" ]]; then - QEC_DECODING_SERVER_PORT="$SERVER_PORT" \ - "$EXE" "${REALTIME_ARGS[@]}" 2>&1 | tee "$REALTIME_LOG" - app_status=${PIPESTATUS[0]} -else - "$EXE" "${REALTIME_ARGS[@]}" 2>&1 | tee "$REALTIME_LOG" - app_status=${PIPESTATUS[0]} -fi -set -e - -if [[ -n "$SERVER_PORT" ]]; then - stop_server -fi - -if [[ "$app_status" -ne 0 ]]; then - echo "FAIL: realtime phase exited with non-zero status ($app_status)" - return_code=1 -fi - -# -------------------------------------------------------------------------- # -# Assertions on the realtime log. -# -------------------------------------------------------------------------- # -echo "" -echo "=== Checking realtime output ===" - -# A non-graphlike DEM handed to pymatching surfaces as "Invalid column in H". -if grep -q "Invalid column in H" "$REALTIME_LOG"; then - echo "FAIL: found 'Invalid column in H' (decoder received a non-graphlike DEM)" - return_code=1 -fi - -# Hard decoder-init / dispatch failures. -if grep -q "terminate called" "$REALTIME_LOG"; then - echo "FAIL: found 'terminate called' (the app aborted)" - return_code=1 -fi -if grep -q "Decoder 0 not found" "$REALTIME_LOG"; then - echo "FAIL: found 'Decoder 0 not found' (decoder was not registered)" - return_code=1 -fi -if grep -q "Error initializing decoders" "$REALTIME_LOG"; then - echo "FAIL: found 'Error initializing decoders'" - return_code=1 -fi - -# A "Number of corrections decoder found:" line MUST be present -- it proves the -# realtime decoding path actually ran to completion. -if ! grep -q "Number of corrections decoder found:" "$REALTIME_LOG"; then - echo "FAIL: missing 'Number of corrections decoder found:' line (decoding did not complete)" - return_code=1 -fi - -# Pull out the residual logical-error count and sanity check it. -num_non_zero_values=$(grep "Number of non-zero values measured :" "$REALTIME_LOG" | awk -F': ' '{print $2}' | tr -d '[:space:]') - -if ! [[ "$num_non_zero_values" =~ ^[0-9]+$ ]]; then - echo "FAIL: 'Number of non-zero values measured' is not a number (got '$num_non_zero_values')" - return_code=1 -elif [[ -n "${SKIP_LOGICAL_ERROR_CEILING:-}" ]]; then - echo "Logical-error ceiling skipped for deterministic concurrency test" -elif [[ "$MULTI_TYPE" -eq 1 ]]; then - echo "Multi-type mode: aggregate ceiling replaced by per-decoder ceilings below" -elif [[ "$num_non_zero_values" -gt "$MAX_NON_ZERO" ]]; then - echo "FAIL: residual logical errors ($num_non_zero_values) exceed ceiling ($MAX_NON_ZERO) -- decoder appears wired-but-wrong" - return_code=1 -else - echo "Residual logical errors: $num_non_zero_values (ceiling $MAX_NON_ZERO) -- OK" -fi - -# Multi-type mode: one report line per patch, matched literally (grep -F -- -# the bracketed label is a regex trap), with a per-type residual ceiling. -if [[ "$MULTI_TYPE" -eq 1 ]]; then - for i in "${!DECODER_TYPES[@]}"; do - t=${DECODER_TYPES[$i]} - line=$(grep -F "decoder[$i] ($t):" "$REALTIME_LOG" || true) - if [[ -z "$line" ]]; then - echo "FAIL: missing per-decoder report line 'decoder[$i] ($t):'" - return_code=1 - continue - fi - errs=$(printf '%s\n' "$line" | sed -n 's/.*logical_errors=\([0-9][0-9]*\)\/[0-9][0-9]*.*/\1/p') - ceil=$MAX_NON_ZERO - if ! [[ "$errs" =~ ^[0-9]+$ ]]; then - echo "FAIL: could not parse logical_errors from: $line" - return_code=1 - elif [[ -n "${SKIP_LOGICAL_ERROR_CEILING:-}" ]]; then - echo "decoder[$i] ($t): logical-error ceiling skipped" - elif [[ "$errs" -gt "$ceil" ]]; then - echo "FAIL: decoder[$i] ($t) residual logical errors ($errs) exceed ceiling ($ceil)" - return_code=1 - else - echo "decoder[$i] ($t): residual logical errors $errs (ceiling $ceil) -- OK" - fi - done -fi - -if [[ -n "${EXPECTED_DECODER_CORRECTIONS:-}" ]]; then - IFS=',' read -r -a expected_corrections <<< \ - "$EXPECTED_DECODER_CORRECTIONS" - if [[ ${#expected_corrections[@]} -ne ${#DECODER_TYPES[@]} ]]; then - echo "FAIL: EXPECTED_DECODER_CORRECTIONS has ${#expected_corrections[@]} entries; expected ${#DECODER_TYPES[@]}" - return_code=1 - else - for i in "${!expected_corrections[@]}"; do - line=$(grep -F "decoder[$i] (${DECODER_TYPES[$i]}):" \ - "$REALTIME_LOG" || true) - got=$(printf '%s\n' "$line" | \ - sed -n 's/.*corrections=\([0-9][0-9]*\),.*/\1/p') - if [[ "$got" != "${expected_corrections[$i]}" ]]; then - echo "FAIL: decoder[$i] corrections='$got'; expected ${expected_corrections[$i]}" - return_code=1 - fi - done - fi -fi - -if [[ -n "$SERVER_PORT" ]]; then - server_dispatches=$(sed -n \ - 's/^QEC_DECODING_SERVER_DISPATCHED count=\([0-9][0-9]*\)$/\1/p' \ - "$SERVER_LOG" | tail -n1) - server_max_concurrent=$(sed -n \ - 's/^QEC_DECODING_SERVER_MAX_CONCURRENT_DECODERS count=\([0-9][0-9]*\)$/\1/p' \ - "$SERVER_LOG" | tail -n1) - minimum_dispatches=$((NUM_SHOTS * ${#DECODER_TYPES[@]} * (NUM_ROUNDS + 3))) - - if ! [[ "$server_dispatches" =~ ^[0-9]+$ ]] || \ - [[ "$server_dispatches" -lt "$minimum_dispatches" ]]; then - echo "FAIL: server dispatch count '$server_dispatches' is below $minimum_dispatches" - return_code=1 - fi - if ! grep -q \ - "External decoding server owns all configured decoder instances" \ - "$REALTIME_LOG"; then - echo "FAIL: external application did not report server-owned decoders" - return_code=1 - fi - if [[ -n "${REQUIRE_DECODER_CONCURRENCY:-}" ]] && \ - { ! [[ "$server_max_concurrent" =~ ^[0-9]+$ ]] || \ - [[ "$server_max_concurrent" -lt "$REQUIRE_DECODER_CONCURRENCY" ]]; }; then - echo "FAIL: server max concurrency '$server_max_concurrent' is below $REQUIRE_DECODER_CONCURRENCY" - return_code=1 - fi - if [[ -n "${EXPECTED_BARRIER_COMPLETIONS:-}" ]]; then - barrier_completions=$(grep -c \ - '^QEC_CONCURRENCY_TEST_BARRIER generation=' "$SERVER_LOG" || true) - if [[ "$barrier_completions" -ne "$EXPECTED_BARRIER_COMPLETIONS" ]]; then - echo "FAIL: barrier completed $barrier_completions times; expected $EXPECTED_BARRIER_COMPLETIONS" - return_code=1 - fi - fi - if [[ -n "${EXPECTED_SERVER_DECODER_CONSTRUCTIONS:-}" ]]; then - server_constructions=$(grep -c \ - '^QEC_CONCURRENCY_TEST_DECODER_CONSTRUCTED$' "$SERVER_LOG" || true) - client_constructions=$(grep -c \ - '^QEC_CONCURRENCY_TEST_DECODER_CONSTRUCTED$' "$REALTIME_LOG" || true) - if [[ "$server_constructions" != \ - "$EXPECTED_SERVER_DECODER_CONSTRUCTIONS" ]]; then - echo "FAIL: server constructed $server_constructions test decoders; expected $EXPECTED_SERVER_DECODER_CONSTRUCTIONS" - return_code=1 - fi - if [[ "$client_constructions" -ne 0 ]]; then - echo "FAIL: external application constructed $client_constructions local decoder instances" - return_code=1 - fi - fi - # Decoder-agnostic execution evidence: the server reports per-session - # counters (QEC_DECODING_SERVER_STATS=1); every decoder must have completed - # exactly one decode per shot. This replaces the removed trt-internal - # QEC_TRT_INFERENCE_EXECUTIONS counter and covers closed-source decoders. - if [[ -n "${REQUIRE_SERVER_DECODE_COUNTS:-}" ]]; then - stats_lines=$(grep -c '^QEC_DECODING_SERVER_DECODER_STATS ' \ - "$SERVER_LOG" || true) - if [[ "$stats_lines" -ne "${#DECODER_TYPES[@]}" ]]; then - echo "FAIL: server reported $stats_lines decoder stats lines; expected ${#DECODER_TYPES[@]}" - return_code=1 - fi - for ((i = 0; i < ${#DECODER_TYPES[@]}; i++)); do - got_decodes=$(sed -n \ - "s/^QEC_DECODING_SERVER_DECODER_STATS id=$i decodes=\([0-9][0-9]*\) .*/\1/p" \ - "$SERVER_LOG" | tail -n1) - if [[ "$got_decodes" != "$NUM_SHOTS" ]]; then - echo "FAIL: decoder[$i] (${DECODER_TYPES[$i]}) completed '$got_decodes' decodes; expected $NUM_SHOTS" - return_code=1 - fi - done - fi - echo "Server evidence: dispatches=$server_dispatches, max_concurrent=$server_max_concurrent" -fi - -# REQUIRE_HOST_MODE (relay trio ctest): assert the realtime session actually -# initialized in HOST dispatch mode, so the test cannot pass vacuously through -# the legacy direct-call path. Needs CUDAQ_LOG_LEVEL=info. -if [[ -n "${REQUIRE_HOST_MODE:-}" ]]; then - if ! grep -q "using HOST dispatch mode" "$REALTIME_LOG"; then - echo "FAIL: 'using HOST dispatch mode' not found (realtime session did not initialize; is CUDAQ_LOG_LEVEL=info set?)" - return_code=1 - fi -fi - -echo "" -if [[ "$return_code" -eq 0 ]]; then - echo "PASS: surface_code-4-yaml ($DECODER_TYPE, d=$DISTANCE) realtime decode succeeded" -else - echo "FAIL: surface_code-4-yaml ($DECODER_TYPE, d=$DISTANCE) test failed" -fi - -exit $return_code diff --git a/libs/qec/unittests/realtime/app_examples/surface_code-4-yaml.cpp b/libs/qec/unittests/realtime/app_examples/surface_code-4-yaml.cpp deleted file mode 100644 index a31dafe33..000000000 --- a/libs/qec/unittests/realtime/app_examples/surface_code-4-yaml.cpp +++ /dev/null @@ -1,1787 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -// For full test script: surface_code-4-yaml-test.sh - -#include "cudaq.h" -#include "cudaq/qec/code.h" -#include "cudaq/qec/codes/surface_code.h" -#include "cudaq/qec/decoder.h" -#include "cudaq/qec/experiments.h" -#include "cudaq/qec/pcm_utils.h" -#include "cudaq/qec/realtime/decoding.h" -#include "cudaq/qec/realtime/decoding_config.h" -#ifdef QEC_APP_EXTERNAL_DECODING_SERVER -#include "cudaq/realtime.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef QEC_APP_EXTERNAL_DECODING_SERVER -namespace { - -class realtime_channel_guard { -public: - realtime_channel_guard() = default; - realtime_channel_guard(const realtime_channel_guard &) = delete; - realtime_channel_guard &operator=(const realtime_channel_guard &) = delete; - - ~realtime_channel_guard() { - if (active_) - cudaq::realtime::finalize(); - } - - void initialize(const char *program) { - const char *port = std::getenv("QEC_DECODING_SERVER_PORT"); - if (!port || port[0] == '\0') - throw std::runtime_error( - "QEC_DECODING_SERVER_PORT is required for external decoding"); - - std::vector args = {program, "--cudaq-device-call=udp", - "udp-host=127.0.0.1", - std::string("udp-port=") + port}; - std::vector argv; - argv.reserve(args.size() + 1); - for (auto &arg : args) - argv.push_back(arg.data()); - argv.push_back(nullptr); - int argc = static_cast(args.size()); - cudaq::realtime::initialize(argc, argv.data()); - active_ = true; - } - -private: - bool active_ = false; -}; - -} // namespace -#endif - -// Host-side decoding API (for syndrome capture) -namespace cudaq::qec::decoding::host { -void _set_syndrome_capture_callback(void (*callback)(const uint8_t *, size_t)); -} - -// Global syndrome capture state for --save_syndrome option. -// -// The live path enqueues a HETEROGENEOUS stream per shot: `num_rounds` uniform -// syndrome rounds of `g_syndrome_bits_per_round` bits each (the prep round plus -// the num_rounds-1 paired rounds), followed by ONE final DATA round of -// `g_data_bits` bits (numData). The capture callback fires once per enqueue, in -// chronological order, so it uses the per-shot enqueue index -// (g_syndrome_count % g_enqueues_per_shot) to decide how many bits to record: -// the syndrome rounds emit g_syndrome_bits_per_round, the last enqueue emits -// g_data_bits. Recording the data round at its true (numData) width is what the -// pre-fix code got wrong -- it truncated those bits to the uniform syndrome -// width and the replayed boundary detectors then disagreed with the live run. -static std::ofstream g_syndrome_output_file; -static std::mutex g_syndrome_file_mutex; -static int g_syndrome_count = 0; -static int g_enqueues_per_shot = 0; -static int g_syndrome_bits_per_round = 0; -static int g_data_bits = 0; - -// Uncomment this to manually inject errors. -// #define MANUALLY_INJECT_ERRORS - -// --------------------------------------------------------------------------- -// Ising-bundle interop (trt+Ising path) -// -// The Ising d/T/Z bundle (generate_test_data.py) ships H_csr.bin/O_csr.bin/ -// priors.bin in *Ising detector order*, plus a D_sparse.txt we generate that -// expresses each Ising detector as a parity over the *cudaqx* live measurement -// buffer. With these, the trt+Ising config carries Ising's exact H/O/priors and -// a D_sparse aligned to Ising's detector rows while reading the cudaqx stream. -// -// Geometry: cudaqx's surface code at orientation XV is identical to the bundle -// geometry (Ising code_rotation string "XV" == first_bulk X, rotated_type V, -// logical_direction XH) under the IDENTITY data and X-ancilla mapping; only the -// Z-ancillas are permuted (a fixed bijection). D_sparse.txt encodes exactly -// that: it takes Ising's detector->measurement map and translates each -// measurement index from Ising's buffer order into cudaqx's buffer order -// (X-ancilla and data identity, Z-ancilla permuted), so every D row reproduces -// one of cudaqx's own detector bits, now in Ising's detector order. -// --------------------------------------------------------------------------- - -// Read a binary-CSR file (rows:u32, cols:u32, nnz:u32, indptr[(rows+1)*i32], -// indices[nnz*i32]) and return it as a -1-terminated sparse row vector -// (each row lists its non-zero column indices, terminated by -1) -- exactly the -// shape pcm_to_sparse_vec produces. Also returns the row/col counts. -static std::vector -read_csr_bin_to_sparse_vec(const std::string &path, std::uint32_t &rows, - std::uint32_t &cols) { - std::ifstream f(path, std::ios::binary); - if (!f) - throw std::runtime_error("Could not open Ising CSR file: " + path); - std::uint32_t nnz = 0; - f.read(reinterpret_cast(&rows), sizeof(rows)); - f.read(reinterpret_cast(&cols), sizeof(cols)); - f.read(reinterpret_cast(&nnz), sizeof(nnz)); - std::vector indptr(rows + 1), indices(nnz); - f.read(reinterpret_cast(indptr.data()), - static_cast(indptr.size() * sizeof(std::int32_t))); - f.read(reinterpret_cast(indices.data()), - static_cast(indices.size() * sizeof(std::int32_t))); - if (!f) - throw std::runtime_error("Truncated Ising CSR file: " + path); - std::vector sparse; - for (std::uint32_t r = 0; r < rows; ++r) { - for (std::int32_t k = indptr[r]; k < indptr[r + 1]; ++k) - sparse.push_back(static_cast(indices[k])); - sparse.push_back(-1); - } - return sparse; -} - -// Read priors.bin (n:u32, n*float64) -> error_rate_vec. -static std::vector read_priors_bin(const std::string &path) { - std::ifstream f(path, std::ios::binary); - if (!f) - throw std::runtime_error("Could not open Ising priors file: " + path); - std::uint32_t n = 0; - f.read(reinterpret_cast(&n), sizeof(n)); - std::vector priors(n); - f.read(reinterpret_cast(priors.data()), - static_cast(priors.size() * sizeof(double))); - if (!f) - throw std::runtime_error("Truncated Ising priors file: " + path); - return priors; -} - -// Read D_sparse.txt -- a whitespace-separated, -1-terminated sparse detector -// matrix. One row per Ising detector; entries are cudaqx live-buffer -// measurement indices (so each row reproduces a cudaqx detector bit, in Ising's -// detector order). Returns the flat -1-terminated vector and the row count. -static std::vector read_D_sparse_txt(const std::string &path, - std::size_t &numRows) { - std::ifstream f(path); - if (!f) - throw std::runtime_error("Could not open Ising D_sparse.txt file: " + path); - std::vector D; - std::int64_t v; - numRows = 0; - while (f >> v) { - D.push_back(v); - if (v == -1) - ++numRows; - } - return D; -} - -// Read the Ising bundle's metadata.txt (key=value lines) and enforce that it -// matches THIS experiment: basis Z, code_rotation XV, and the same distance / -// n_rounds. This is the semantic guard the dimensional checks cannot provide -- -// so the bundle's basis and orientation match the experiment. -static void enforce_ising_metadata(const std::string &bundle, int distance, - std::size_t numRounds) { - std::ifstream f(bundle + "/metadata.txt"); - if (!f) - throw std::runtime_error("Ising bundle missing metadata.txt: " + bundle + - "/metadata.txt"); - std::vector> kv; - std::string line; - while (std::getline(f, line)) { - while (!line.empty() && (line.back() == '\r' || line.back() == ' ')) - line.pop_back(); - auto eq = line.find('='); - if (eq != std::string::npos) - kv.emplace_back(line.substr(0, eq), line.substr(eq + 1)); - } - auto require = [&](const std::string &key, const std::string &want) { - for (const auto &p : kv) - if (p.first == key) { - if (p.second != want) - throw std::runtime_error("Ising bundle " + bundle + - ": metadata.txt " + key + "='" + p.second + - "' != required '" + want + "'"); - return; - } - throw std::runtime_error("Ising bundle " + bundle + - ": metadata.txt missing key '" + key + "'"); - }; - require("basis", "Z"); - require("code_rotation", "XV"); - require("distance", std::to_string(distance)); - require("n_rounds", std::to_string(numRounds)); -} - -// Flatten cudaqx's m2d into the -1-terminated sparse detector matrix, in -// cudaqx detector order (D_sparse[i] == m2d.rows[i]). This is the -// self-consistent D for the cudaqx-native decoders (pymatching / nv-qldpc), -// which carry cudaqx's own dem_gen_circuit H/O. It references the same -// chronological measurement indices the live path enqueues (385 bits for -// d7/T7), including the final 49 data measurements used by the boundary -// detectors. -static std::vector -build_cudaqx_D_sparse(const cudaq::M2DSparseMatrix &m2d) { - std::vector D; - for (const auto &row : m2d.rows) { - for (auto meas : row) - D.push_back(static_cast(meas)); - D.push_back(-1); - } - return D; -} - -// One decoder entry per patch: entry i (decoder id i) gets decoder_types[i]. -// Entries carry decoder-appropriate representations of one source DEM per -// patch, allowing each logical patch to use its own physical error rate: -// matching-family entries (pymatching / trt_decoder) the decomposed graph-like -// columns (`dem`; each component carries the parent instruction's probability -// -- the documented lossy approximation matching requires), BP-family entries -// (nv-qldpc-decoder) the joint hyperedge columns (`dem_bp`). Detector -// geometry is identical across entries, and the default D_sparse mappings are -// identical too; a trt+Ising entry substitutes an Ising-ordered D over the -// same measurement span. The error-column representations are intentionally -// not probabilistically identical. -void save_dem_to_file( - const std::vector &dems, - const std::vector &dems_bp, - std::string dem_filename, const std::vector &decoder_types, - bool use_relay_bp, const std::string &onnx_path, - const cudaq::M2DSparseMatrix &m2d, const std::string &ising_bundle, - int distance, std::size_t numRounds) { - cudaq::qec::decoding::config::multi_decoder_config multi_config; - for (uint64_t i = 0; i < decoder_types.size(); i++) { - const std::string &decoder_type = decoder_types[i]; - const auto &edem = - (decoder_type == "nv-qldpc-decoder") ? dems_bp[i] : dems[i]; - cudaq::qec::decoding::config::decoder_config config; - config.id = i; - config.type = decoder_type; - config.block_size = edem.num_error_mechanisms(); - config.syndrome_size = edem.num_detectors(); - config.H_sparse = cudaq::qec::pcm_to_sparse_vec(edem.detector_error_matrix); - config.O_sparse = - cudaq::qec::pcm_to_sparse_vec(edem.observables_flips_matrix); - // Default D == cudaqx's m2d (cudaqx detector order), self-consistent with - // the dem_gen_circuit H/O above and with the full 385-bit measurement - // stream the live path enqueues. The trt+Ising branch below overrides this - // with the Ising-ordered D_sparse.txt to match the Ising H/O. - config.D_sparse = build_cudaqx_D_sparse(m2d); - - if (decoder_type == "nv-qldpc-decoder") { - cudaqx::heterogeneous_map nv_args; - - // Basic settings - nv_args.insert("use_sparsity", true); - nv_args.insert("error_rate_vec", edem.error_rates); - nv_args.insert("max_iterations", 50); - - if (use_relay_bp) { - nv_args.insert("bp_method", 3); // min-sum+dmem (required for relay) - nv_args.insert("composition", 1); // Enable sequential relay - nv_args.insert("gamma0", 0.0); // Initial gamma value - nv_args.insert("clip_value", 200.0); - nv_args.insert("repeatable", true); - cudaqx::heterogeneous_map srelay_args; - srelay_args.insert("pre_iter", std::size_t{5}); - srelay_args.insert("num_sets", std::size_t{10}); - srelay_args.insert("stopping_criterion", "All"); - srelay_args.insert("stop_nconv", std::size_t{1}); - nv_args.insert("srelay_config", srelay_args); - nv_args.insert("gamma_dist", std::vector{0.1, 0.2}); - } else { - // OSD post-processor - nv_args.insert("use_osd", true); - nv_args.insert("osd_order", 60); - nv_args.insert("osd_method", 3); - } - config.decoder_custom_args = nv_args; - } else if (decoder_type == "pymatching") { - cudaqx::heterogeneous_map pm_args; - pm_args.insert("merge_strategy", "smallest_weight"); - pm_args.insert("error_rate_vec", edem.error_rates); - config.decoder_custom_args = pm_args; - } else if (decoder_type == "trt_decoder") { - cudaqx::heterogeneous_map trt_args; - // The TensorRT predecoder model is supplied through the saved decoder - // config so the same ONNX path is used after reload. - trt_args.insert("onnx_load_path", onnx_path); - trt_args.insert("batch_size", std::size_t{1}); - trt_args.insert("use_cuda_graph", true); - trt_args.insert("global_decoder", "pymatching"); - - cudaqx::heterogeneous_map pm_args; - pm_args.insert("merge_strategy", "smallest_weight"); - - if (!ising_bundle.empty()) { - // Enforce the bundle's semantics match this experiment (basis Z, - // code_rotation XV, same d / n_rounds) before trusting its matrices. - enforce_ising_metadata(ising_bundle, distance, numRounds); - // trt+Ising path: carry the Ising d/T/Z model. H/O/priors come from - // the Ising bundle (Ising detector order); D_sparse (D_sparse.txt) - // expresses each Ising detector as a parity over the cudaqx live - // measurement buffer, so the live stream feeds Ising's detectors in - // Ising's row order. - std::uint32_t hRows = 0, hCols = 0, oRows = 0, oCols = 0; - config.H_sparse = read_csr_bin_to_sparse_vec( - ising_bundle + "/H_csr.bin", hRows, hCols); - config.O_sparse = read_csr_bin_to_sparse_vec( - ising_bundle + "/O_csr.bin", oRows, oCols); - auto priors = read_priors_bin(ising_bundle + "/priors.bin"); - std::size_t dRows = 0; - config.D_sparse = - read_D_sparse_txt(ising_bundle + "/D_sparse.txt", dRows); - - if (hRows != m2d.rows.size()) - throw std::runtime_error("Ising H rows (" + std::to_string(hRows) + - ") != cudaqx m2d detectors (" + - std::to_string(m2d.rows.size()) + ")"); - if (dRows != m2d.rows.size()) - throw std::runtime_error("D_sparse.txt rows (" + - std::to_string(dRows) + - ") != cudaqx m2d detectors (" + - std::to_string(m2d.rows.size()) + ")"); - if (hCols != oCols || hCols != priors.size()) - throw std::runtime_error("Ising H/O/priors column counts disagree"); - - config.syndrome_size = hRows; - config.block_size = hCols; - pm_args.insert("error_rate_vec", priors); - printf("trt+Ising: loaded Ising bundle '%s' (H %ux%u, O %u rows, " - "priors %zu); D_sparse from D_sparse.txt (%zu detectors)\n", - ising_bundle.c_str(), hRows, hCols, oRows, priors.size(), dRows); - } else { - pm_args.insert("error_rate_vec", edem.error_rates); - } - trt_args.insert("global_decoder_params", pm_args); - - config.decoder_custom_args = trt_args; - } - - multi_config.decoders.push_back(config); - } - std::string config_str = multi_config.to_yaml_str(200); - std::ofstream config_file(dem_filename); - config_file << config_str; - config_file.close(); - if (!config_file) - throw std::runtime_error("failed to write decoder config: " + dem_filename); - std::string types_str; - for (std::size_t i = 0; i < decoder_types.size(); ++i) - types_str += (i ? "," : "") + decoder_types[i]; - printf("Saved %s config to file: %s\n", types_str.c_str(), - dem_filename.c_str()); - return; -} - -// Parse and validate a saved decoder config: geometry guards, per-id types, -// and the reconstructed DEM. Does NOT construct decoders -- instantiation is -// the serving side's job (this process for in-process serving; the decoding -// server for a server-served run). -cudaq::qec::decoding::config::multi_decoder_config -load_decoder_config(const std::string &dem_filename, - cudaq::qec::detector_error_model &dem, uint64_t numLogical, - uint64_t &measSpan, - std::vector &decoder_types_out) { - printf("load_decoder_config: Loading dem from file: %s\n", - dem_filename.c_str()); - // Read dem_filename into a std::string - std::ifstream dem_file(dem_filename); - if (!dem_file) - throw std::runtime_error("could not open decoder config file: " + - dem_filename); - std::string dem_str((std::istreambuf_iterator(dem_file)), - std::istreambuf_iterator()); - auto config = - cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( - dem_str); - if (numLogical != config.decoders.size()) - throw std::runtime_error("numLogical [" + std::to_string(numLogical) + - "] != config.decoders.size() [" + - std::to_string(config.decoders.size()) + "]"); - // Validate EVERY decoder config, mirroring the runtime's per-patch loop in - // demo_circuit_qpu: the runtime requests one correction from each decoder, - // so each decoder must carry exactly ONE observable and all decoders must - // share the same detector/measurement geometry. Entries may differ in TYPE - // (and, for BP-family entries, in error-column factorization / block_size); - // the geometry checks below are what bind them to one experiment. - auto span_of = [](const std::vector &sparse) { - std::int64_t m = -1; - for (auto v : sparse) - if (v > m) - m = v; - return static_cast(m + 1); - }; - const auto &d0 = config.decoders[0]; - const uint64_t span0 = span_of(d0.D_sparse); - decoder_types_out.assign(config.decoders.size(), std::string()); - for (size_t k = 0; k < config.decoders.size(); ++k) { - const auto &dc = config.decoders[k]; - if (dc.id < 0 || static_cast(dc.id) >= config.decoders.size() || - !decoder_types_out[dc.id].empty()) - throw std::runtime_error( - "loaded config decoder ids must be unique and dense in [0, " + - std::to_string(config.decoders.size()) + "); entry " + - std::to_string(k) + " has id " + std::to_string(dc.id)); - decoder_types_out[dc.id] = dc.type; - const size_t nobs = std::count(dc.O_sparse.begin(), dc.O_sparse.end(), -1); - if (nobs != 1) - throw std::runtime_error( - "loaded config decoder " + std::to_string(k) + " has " + - std::to_string(nobs) + - " observables; expected 1 (one observable per single-patch " - "surface-code decoder)"); - if (dc.syndrome_size != d0.syndrome_size || span_of(dc.D_sparse) != span0) - throw std::runtime_error( - "loaded config decoder " + std::to_string(k) + - " geometry (detectors " + std::to_string(dc.syndrome_size) + - ", measurement span " + std::to_string(span_of(dc.D_sparse)) + - ") differs from decoder 0 (detectors " + - std::to_string(d0.syndrome_size) + ", span " + std::to_string(span0) + - "); all decoders must share one experiment geometry"); - } - - auto decoder_config = d0; - dem.detector_error_matrix = cudaq::qec::pcm_from_sparse_vec( - decoder_config.H_sparse, decoder_config.syndrome_size, - decoder_config.block_size); - size_t num_observables = std::count(decoder_config.O_sparse.begin(), - decoder_config.O_sparse.end(), -1); - dem.observables_flips_matrix = cudaq::qec::pcm_from_sparse_vec( - decoder_config.O_sparse, num_observables, decoder_config.block_size); - // The runtime decodes once the enqueued measurement buffer is full; that span - // is max(D_sparse)+1. Surface it so the caller can bind the config to this - // experiment's geometry (see the load_dem check in demo_circuit_host). - measSpan = span0; - - std::string types_str; - for (std::size_t i = 0; i < decoder_types_out.size(); ++i) - types_str += (i ? "," : "") + decoder_types_out[i]; - printf("Loaded %s config from file: %s\n", types_str.c_str(), - dem_filename.c_str()); - return config; -} - -std::vector get_stab_cnot_schedule(char stab_type, int distance) { - // Build the stabilizer CNOT schedule from an XV-oriented grid (the - // predecoder's training orientation). - cudaq::qec::surface_code::stabilizer_grid grid( - distance, cudaq::qec::surface_code::sc_orientation::XV); - if (stab_type != 'X' && stab_type != 'Z') { - throw std::runtime_error( - "get_stab_cnot_schedule: Invalid stabilizer type. Must be 'X' or 'Z'."); - } - // CNOT pairs ordered by timestep within each stabilizer, so that mid-round - // ancilla (hook) errors land perpendicular to the logical operators. - // Stabilizer indices match the sorted parity-matrix rows and hence the - // ancilla indexing. - return stab_type == 'X' ? grid.get_cnot_schedule_pairs_x() - : grid.get_cnot_schedule_pairs_z(); -} - -// Per-stabilizer data-qubit supports, ordered to match the ancilla measurement -// order produced by get_stab_cnot_schedule(stab_type, ...). The supports are -// derived from the same schedule matrix as the CNOT pairs (row s = ancilla s), -// so support[s] lines up with ancilla[s] in se_{x,z}_ft's measurement vector -// by construction rather than via a parallel sort. This is what the Ising -// MemoryCircuit boundary detectors pair against (a stabilizer's data support -// XOR-ed with that stabilizer's last ancilla measurement). Returns a flat -// vector of data-qubit indices plus per-stabilizer offsets (offsets has size -// num_stabs+1; support s spans [offsets[s], offsets[s+1])), the same -// flat+offset pattern as cnot_schedZ_flat. -void get_stab_data_supports(char stab_type, int distance, - std::vector &supports_flat, - std::vector &supports_offsets) { - cudaq::qec::surface_code::stabilizer_grid grid( - distance, cudaq::qec::surface_code::sc_orientation::XV); - if (stab_type != 'X' && stab_type != 'Z') { - throw std::runtime_error( - "get_stab_data_supports: Invalid stabilizer type. Must be 'X' or 'Z'."); - } - auto sched = stab_type == 'X' ? grid.get_cnot_schedule_x() - : grid.get_cnot_schedule_z(); - supports_flat.clear(); - supports_offsets.clear(); - supports_offsets.push_back(0); - for (std::size_t s = 0; s < sched.shape()[0]; ++s) { - for (std::size_t d = 0; d < sched.shape()[1]; ++d) { - if (sched.at({s, d}) != 0) - supports_flat.push_back(d); - } - supports_offsets.push_back(supports_flat.size()); - } -} - -namespace cudaq::qec::qpu { - -__qpu__ void spam_error(cudaq::qec::patch logicalQubit, double p_spam_data, - double p_spam_ancx, double p_spam_ancz) { - for (std::size_t i = 0; i < logicalQubit.data.size(); i++) { - cudaq::apply_noise(p_spam_data, - logicalQubit.data[i]); - } - for (std::size_t i = 0; i < logicalQubit.ancx.size(); i++) { - cudaq::apply_noise(p_spam_ancx, - logicalQubit.ancx[i]); - } - for (std::size_t i = 0; i < logicalQubit.ancz.size(); i++) { - cudaq::apply_noise(p_spam_ancz, - logicalQubit.ancz[i]); - } -} - -__qpu__ std::vector -extract_z_syndrome(cudaq::qec::patch logicalQubit, - const std::vector &cnot_sched) { - for (std::size_t i = 0; i < cnot_sched.size(); i += 2) { - cudaq::x(logicalQubit.data[cnot_sched[i + 1]], - logicalQubit.ancz[cnot_sched[i]]); - } - auto results = mz(logicalQubit.ancz); - for (std::size_t i = 0; i < logicalQubit.ancz.size(); i++) - reset(logicalQubit.ancz[i]); - return results; -} - -__qpu__ std::vector -extract_x_syndrome(cudaq::qec::patch logicalQubit, - const std::vector &cnot_sched) { - h(logicalQubit.ancx); - for (std::size_t i = 0; i < cnot_sched.size(); i += 2) { - cudaq::x(logicalQubit.ancx[cnot_sched[i]], - logicalQubit.data[cnot_sched[i + 1]]); - } - h(logicalQubit.ancx); - auto results = mz(logicalQubit.ancx); - for (std::size_t i = 0; i < logicalQubit.ancx.size(); i++) - reset(logicalQubit.ancx[i]); - return results; -} - -__qpu__ void -measure_syndrome_round(cudaq::qec::patch logicalQubit, - const std::vector &cnot_schedX_flat, - const std::vector &cnot_schedZ_flat, - std::vector &combined_syndrome) { - // Measure X-ancillas then Z-ancillas (combined layout [X..., Z...]). The - // shared helper keeps the DEM and the live enqueue path on the same ordering. - auto syndrome_x = extract_x_syndrome(logicalQubit, cnot_schedX_flat); - auto syndrome_z = extract_z_syndrome(logicalQubit, cnot_schedZ_flat); - int i = 0; - for (auto s : syndrome_x) - combined_syndrome[i++] = s; - for (auto s : syndrome_z) - combined_syndrome[i++] = s; -} - -// Run ONE syndrome-extraction round on one patch and (optionally) enqueue it -// to that patch's decoder. The round loop lives in demo_circuit_qpu so the -// patches can be interleaved round-major. round_counter is the per-patch -// monotonic enqueue index (0 = lock-in, 1..T-1 paired, T = data round): -// logging-only on the in-process path, the round counter on the -// decoding-server wire. -__qpu__ void -syndrome_round_once(cudaq::qview<> data, cudaq::qview<> xstab_anc, - cudaq::qview<> zstab_anc, - const std::vector &cnot_schedX_flat, - const std::vector &cnot_schedZ_flat, - bool do_spam, double p_spam, bool do_enqueue, - int logical_qubit_idx, std::uint64_t round_counter) { - patch logical(data, xstab_anc, zstab_anc); - std::vector combined_syndrome(xstab_anc.size() + - zstab_anc.size()); - // Inject errors BEFORE the measurement, matching the DEM kernel's - // spam-then-measure placement. - if (do_spam) - spam_error(logical, p_spam, 0.0, 0.0); - measure_syndrome_round(logical, cnot_schedX_flat, cnot_schedZ_flat, - combined_syndrome); - if (do_enqueue) - cudaq::qec::decoding::enqueue_syndromes( - /*decoder_id=*/logical_qubit_idx, combined_syndrome, round_counter); -} - -__qpu__ std::vector demo_circuit_qpu( - bool allow_device_calls, - const cudaq::qec::code::one_qubit_encoding &statePrep, std::size_t numData, - std::size_t numAncx, std::size_t numAncz, std::size_t numRounds, - std::size_t numLogical, const std::vector &cnot_schedX_flat, - const std::vector &cnot_schedZ_flat, - const std::vector &p_spam_per_patch, bool apply_corrections) { - // ret[i] = patch i's final data bits (numData bits, enforced < 64 in - // main()); ret[numLogical] = per-patch correction bitmask (bit i set iff - // decoder i predicted a logical flip). - std::vector ret(numLogical + 1); - for (std::size_t k = 0; k <= numLogical; k++) - ret[k] = 0; - - // Reset the decoder - if (allow_device_calls) { - for (int i = 0; i < numLogical; i++) { - cudaq::qec::decoding::reset_decoder(/*decoder_id=*/i); - } - } - - // Allocate the data and ancilla qubits - cudaq::qvector data(numLogical * numData), xstab_anc(numLogical * numAncx), - zstab_anc(numLogical * numAncz); - - // Call state prep - for (int i = 0; i < numLogical; i++) { - auto subData = data.slice(i * numData, numData); - auto subXstab_anc = xstab_anc.slice(i * numAncx, numAncx); - auto subZstab_anc = zstab_anc.slice(i * numAncz, numAncz); - patch logical(subData, subXstab_anc, subZstab_anc); - statePrep(logical); - } - - // Round 0 (lock-in) for every patch: no spam; pins the stabilizer - // eigenvalues after prep. - for (int i = 0; i < numLogical; i++) { - syndrome_round_once( - data.slice(i * numData, numData), xstab_anc.slice(i * numAncx, numAncx), - zstab_anc.slice(i * numAncz, numAncz), cnot_schedX_flat, - cnot_schedZ_flat, /*do_spam=*/false, p_spam_per_patch[i], - /*do_enqueue=*/allow_device_calls, i, - /*round_counter=*/0); - } - - // Paired rounds, ROUND-MAJOR: every patch measures round r before any patch - // measures round r+1, so the decoders' enqueue streams interleave - // (0,1,...,N-1, 0,1,...). Each patch's own stream (spam BEFORE each paired - // round, then measure) is identical to the DEM kernel's schedule; only the - // cross-patch interleaving -- which no decoder observes -- differs from a - // patch-major loop. This example decodes ONE volume of num_rounds rounds (no - // sliding windows), so each syndrome is enqueued EXACTLY ONCE. - for (std::size_t round = 0; round + 1 < numRounds; round++) { - for (int i = 0; i < numLogical; i++) { -#ifdef MANUALLY_INJECT_ERRORS - if (round == 0) { - // Inject a single error - cudaq::x(data[i * numData + 3]); - } -#endif - syndrome_round_once(data.slice(i * numData, numData), - xstab_anc.slice(i * numAncx, numAncx), - zstab_anc.slice(i * numAncz, numAncz), - cnot_schedX_flat, cnot_schedZ_flat, /*do_spam=*/true, - p_spam_per_patch[i], - /*do_enqueue=*/allow_device_calls, i, - /*round_counter=*/round + 1); - } - } - - // Final data round per patch: measure the data qubits in the logical (Z) - // basis and enqueue them as the LAST enqueue of that patch's stream, - // completing the num_rounds*(numAncx+numAncz)+numData-bit buffer the DEM was - // built under (385 at d7/T7). The decoder's boundary detectors XOR these - // data bits against the final-round Z-ancilla measurements. - for (int i = 0; i < numLogical; i++) { - auto subData = data.slice(i * numData, numData); - auto subMeas = mz(subData); - if (allow_device_calls) - cudaq::qec::decoding::enqueue_syndromes(/*decoder_id=*/i, subMeas, - /*round_counter=*/numRounds); - ret[i] = cudaq::to_integer(cudaq::to_bools(subMeas)); - } - - // Apply each decoder's correction classically: a transversal X on the data - // flips every measured data bit, so XOR the low-numData mask into that - // patch's word when its decoder predicts a logical flip. This matches the - // physical-X-then-measure result but lets us enqueue the (uncorrected) data - // bits first. - if (allow_device_calls && apply_corrections) { - for (int i = 0; i < numLogical; i++) { - auto correction_result = cudaq::qec::decoding::get_corrections( - /*decoder_id=*/i, /*return_size=*/1, /*reset=*/false); - if (correction_result[0] != 0) { - ret[numLogical] |= (1ull << i); - std::uint64_t mask = (1ull << numData) - 1; - ret[i] = ret[i] ^ mask; - } - } - } - return ret; -} - -// DEM-generation kernel. The example uses two kernels that share one syndrome- -// extraction helper (measure_syndrome_round): dem_gen_circuit (this one) is -// sampled by dem_from_kernel to build the DEM, so it is a pure detector- -// annotated circuit with no decoder calls; demo_circuit_qpu runs the live -// experiment and makes the decoder RPC calls (enqueue_syndromes / -// get_corrections). The two kernels need to stay in lockstep on the measurement -// schedule (same rounds, X-then-Z order, spam-then-measure placement). -// -// dem_gen_circuit declares detectors; the layout is basis Z, orientation XV: -// - Block 0 (numAncz prep singles): one single-term detector per round-0 -// Z-ancilla (deterministic after prep0 |0>_L). -// - Blocks 1+2 (paired): `pairedRounds` rounds, each pairing this round's -// full syndrome [X..., Z...] against the previous round's, via -// cudaq::detectors(prev, curr) (numAncx + numAncz detectors per round). -// - Block 3 (numAncz boundary): one detector per Z-stabilizer s, XOR-ing the -// final data measurements in that stabilizer's support with the -// last-round Z-ancilla measurement for that stabilizer. -__qpu__ void -dem_gen_circuit(const cudaq::qec::code::one_qubit_encoding &statePrep, - std::size_t numData, std::size_t numAncx, std::size_t numAncz, - std::size_t pairedRounds, - const std::vector &cnot_schedX_flat, - const std::vector &cnot_schedZ_flat, double p_spam, - const std::vector &z_logical_indices, - const std::vector &z_supports_flat, - const std::vector &z_supports_offsets) { - cudaq::qvector data(numData), xstab_anc(numAncx), zstab_anc(numAncz); - patch logical(data, xstab_anc, zstab_anc); - - statePrep(logical); - - std::vector prev(numAncz + numAncx); - measure_syndrome_round(logical, cnot_schedX_flat, cnot_schedZ_flat, prev); - - // Block 0: prep singles on the round-0 Z-ancillas. Post-flip the combined - // syndrome is [X(numAncx), Z(numAncz)], so the Z-ancillas are - // prev[numAncx..]. - for (std::size_t k = 0; k < numAncz; ++k) - cudaq::detector(prev[numAncx + k]); - - // Blocks 1+2: paired cross-round detectors. Exactly `pairedRounds` rounds - // (the middle rounds plus the final round), giving - // pairedRounds * (numAncx + numAncz) detectors. - for (std::size_t round = 0; round < pairedRounds; ++round) { - spam_error(logical, /*p_spam_data=*/p_spam, /*p_spam_ancx=*/0.0, - /*p_spam_ancz=*/0.0); - std::vector curr(numAncz + numAncx); - measure_syndrome_round(logical, cnot_schedX_flat, cnot_schedZ_flat, curr); - cudaq::detectors(prev, curr); - prev = curr; - } - - auto dataM = mz(data); - - // Block 3: boundary detectors. For each Z-stabilizer s, XOR its data-qubit - // support (from the final data measurement) with that stabilizer's last-round - // Z-ancilla measurement (prev still holds the final round syndrome). - for (std::size_t s = 0; s + 1 < z_supports_offsets.size(); ++s) { - std::vector stab_data(z_supports_offsets[s + 1] - - z_supports_offsets[s]); - std::size_t j = 0; - for (std::size_t t = z_supports_offsets[s]; t < z_supports_offsets[s + 1]; - ++t) - stab_data[j++] = dataM[z_supports_flat[t]]; - cudaq::detector(stab_data, prev[numAncx + s]); - } - - std::vector zlog(z_logical_indices.size()); - for (std::size_t k = 0; k < z_logical_indices.size(); ++k) - zlog[k] = dataM[z_logical_indices[k]]; - cudaq::logical_observable(zlog, /*observable_index=*/0); -} -} // namespace cudaq::qec::qpu - -void demo_circuit_host(const cudaq::qec::code &code, int distance, - const std::vector &p_spam_per_patch, - cudaq::qec::operation statePrep, std::size_t numShots, - std::size_t numRounds, std::size_t numLogical, - std::string dem_filename, bool save_dem, bool load_dem, - const std::vector &decoder_types, - bool save_syndrome = false, bool load_syndrome = false, - std::string syndrome_filename = "", - bool use_relay_bp = false, std::string onnx_path = "", - std::string ising_bundle = "") { - if (!code.contains_operation(statePrep)) - throw std::runtime_error( - "sample_memory_circuit_error - requested state prep kernel not found."); - - auto &prep = - code.get_operation(statePrep); - - auto numData = code.get_num_data_qubits(); - auto numAncx = code.get_num_ancilla_x_qubits(); - auto numAncz = code.get_num_ancilla_z_qubits(); - - auto cnot_schedX_flat = get_stab_cnot_schedule('X', distance); - auto cnot_schedZ_flat = get_stab_cnot_schedule('Z', distance); - std::size_t numSyndromesPerRound = 0; - - printf("cnot_schedX_flat: "); - // Put a comma in between each pair of elements - for (std::size_t i = 0; i < cnot_schedX_flat.size(); i += 2) - printf("%lu %lu, ", cnot_schedX_flat[i], cnot_schedX_flat[i + 1]); - printf("\n"); - printf("cnot_schedZ_flat: "); - for (std::size_t i = 0; i < cnot_schedZ_flat.size(); i += 2) - printf("%lu %lu, ", cnot_schedZ_flat[i], cnot_schedZ_flat[i + 1]); - printf("\n"); - - cudaq::noise_model noise; - - // First get the DEM. - cudaq::qec::detector_error_model dem; - // Per-id decoder types read back from the YAML (run phase), for the - // per-decoder report lines. - std::vector loaded_types; - if (load_dem) { - uint64_t measSpan = 0; - auto decoder_config = load_decoder_config(dem_filename, dem, numLogical, - measSpan, loaded_types); - const auto numDetectors = dem.detector_error_matrix.shape()[0]; - const auto fullSyndromesPerRound = numAncx + numAncz; - // Bind the loaded config to this experiment's geometry: the detector count - // and the measurement-buffer span (max(D_sparse)+1) must both match the - // distance and num_rounds being run. - const auto expectedDetectors = - static_cast(numRounds) * fullSyndromesPerRound; - const auto expectedSpan = expectedDetectors + numData; - if (numDetectors != expectedDetectors) - throw std::runtime_error( - "Loaded DEM detector count (" + std::to_string(numDetectors) + - ") does not match this experiment (d=" + std::to_string(distance) + - ", num_rounds=" + std::to_string(numRounds) + " -> expected " + - std::to_string(expectedDetectors) + ")"); - if (measSpan != expectedSpan) - throw std::runtime_error( - "Loaded config measurement-buffer span max(D_sparse)+1 (" + - std::to_string(measSpan) + - ") does not match this experiment's enqueued bits (expected " + - std::to_string(expectedSpan) + - " = num_rounds*(numAncx+numAncz)+numData for d=" + - std::to_string(distance) + - ", num_rounds=" + std::to_string(numRounds) + - "); the YAML was generated for a different geometry"); - numSyndromesPerRound = fullSyndromesPerRound; - // Construct local decoders only for the in-process executable. The external - // executable still performs all geometry validation above, but the server - // owns the decoder instances described by this same configuration. -#ifdef QEC_APP_EXTERNAL_DECODING_SERVER - printf("External decoding server owns all configured decoder instances\n"); -#else - int rc = cudaq::qec::decoding::config::configure_decoders(decoder_config); - if (rc != 0) - throw std::runtime_error("configure_decoders failed (status " + - std::to_string(rc) + ")"); -#endif - } else { - for (std::size_t i = 0; i < p_spam_per_patch.size(); ++i) - if (p_spam_per_patch[i] == 0.0) - throw std::runtime_error("--p_spam must be > 0 to generate a DEM " - "(--p_spam_per_patch patch " + - std::to_string(i) + - " has a zero-noise model)"); - - // Left-column data qubits are the Z logical for XV (and the legacy ZH). - // This must track the code's Z observable (code.get_observables_z()); if - // the orientation changes to XH/ZV the Z logical moves to the top row. - std::vector z_logical_indices; - for (int i = 0; i < distance; ++i) - z_logical_indices.push_back(static_cast(i) * distance); - - // Z-stabilizer data-qubit supports, aligned to the Z-ancilla measurement - // order (support[s] <-> Z-ancilla[s]). Used to build the boundary - // detectors (basis Z). - std::vector z_supports_flat, z_supports_offsets; - get_stab_data_supports('Z', distance, z_supports_flat, z_supports_offsets); - - // The DEM detector layout is: - // numAncz prep singles + pairedRounds * (numAncx + numAncz) paired - // + numAncz boundary. - // This example decodes ONE volume of num_rounds rounds. Ising's n_rounds - // counts the state-prep round and the final logical-measurement round, so - // it has (n_rounds - 1) paired transitions. Here the round-0 syndrome is - // the prep round and the volume covers num_rounds syndrome rounds total, so - // there are (num_rounds - 1) paired rounds (the 5 middle + 1 final at d=7, - // num_rounds=7). - const std::size_t pairedRounds = static_cast(numRounds) - 1; - - auto contains_type = [&](const char *t) { - return std::find(decoder_types.begin(), decoder_types.end(), t) != - decoder_types.end(); - }; - // Matching decoders (pymatching / trt_decoder's global pymatching) need - // the DEM sampled with decomposition suggestions; BP (nv-qldpc-decoder) - // natively decodes the undecomposed hyperedge columns. - const bool haveMatching = - contains_type("pymatching") || contains_type("trt_decoder"); - const bool haveBp = contains_type("nv-qldpc-decoder"); - const bool decompose_errors = haveMatching; - cudaq::M2DSparseMatrix m2d; - std::vector patch_dems; - std::vector patch_dems_undecomposed; - patch_dems.reserve(numLogical); - patch_dems_undecomposed.reserve(numLogical); - const bool dual_parse = haveMatching && haveBp; - for (std::size_t patch = 0; patch < numLogical; ++patch) { - cudaq::M2DSparseMatrix patch_m2d; - cudaq::M2OSparseMatrix patch_m2o; - const std::string dem_text = cudaq::dem_from_kernel( - cudaq::qec::qpu::dem_gen_circuit, &noise, - cudaq::dem_options{.decompose_errors = decompose_errors}, patch_m2d, - patch_m2o, prep, numData, numAncx, numAncz, pairedRounds, - cnot_schedX_flat, cnot_schedZ_flat, p_spam_per_patch[patch], - z_logical_indices, z_supports_flat, z_supports_offsets); - if (patch == 0) - m2d = patch_m2d; - else if (patch_m2d.rows != m2d.rows) - throw std::runtime_error( - "per-patch DEMs produced different measurement mappings"); - - auto patch_dem = cudaq::qec::dem_from_stim_text( - dem_text, /*use_decomp_suggestions=*/decompose_errors); - patch_dems.push_back(patch_dem); - patch_dems_undecomposed.push_back( - dual_parse ? cudaq::qec::dem_from_stim_text( - dem_text, /*use_decomp_suggestions=*/false) - : patch_dem); - } - dem = patch_dems.front(); - - numSyndromesPerRound = numAncx + numAncz; - printf("numSyndromesPerRound: %ld\n", numSyndromesPerRound); - - printf("dem.detector_error_matrix:\n"); - dem.detector_error_matrix.dump_bits(); - printf("dem.observables_flips_matrix:\n"); - dem.observables_flips_matrix.dump_bits(); - - if (save_dem) { - save_dem_to_file(patch_dems, patch_dems_undecomposed, dem_filename, - decoder_types, use_relay_bp, onnx_path, m2d, - ising_bundle, distance, numRounds); - return; - } - } - - // Detector count for one volume of num_rounds rounds: - // numAncz (prep singles) + (num_rounds-1)*(numAncx+numAncz) (paired) - // + numAncz (boundary). - // This is divisible by numSyndromesPerRound (= numAncx+numAncz) into exactly - // num_rounds rounds ONLY because numAncx == numAncz for the rotated surface - // code (so the prep block numAncz and the boundary block numAncz together - // make one full numAncx+numAncz round). If that ever stops holding, this - // divisibility / round-count bookkeeping must be re-derived. - if (dem.detector_error_matrix.shape()[0] % numSyndromesPerRound != 0) { - throw std::runtime_error("Num syndromes per round is not a divisor of " - "the number of syndrome measurements"); - } - size_t numRoundsOfSyndromData = - dem.detector_error_matrix.shape()[0] / numSyndromesPerRound; - - if (numRoundsOfSyndromData != static_cast(numRounds)) { - throw std::runtime_error("Num rounds of syndrome data [" + - std::to_string(numRoundsOfSyndromData) + - "] is not equal to num_rounds [" + - std::to_string(numRounds) + "]"); - } - - // Setup syndrome capture if requested (--save_syndrome option) - if (save_syndrome) { - if (syndrome_filename.empty()) { - printf("Error: --save_syndrome requires a filename argument\n"); - return; - } - - g_syndrome_output_file.open(syndrome_filename, - std::ios::out | std::ios::trunc); - if (!g_syndrome_output_file) { - printf("Error: Could not open syndrome file for writing: %s\n", - syndrome_filename.c_str()); - return; - } - - // Per-shot enqueue structure: num_rounds uniform syndrome rounds - // (numSyndromesPerRound bits each) + 1 final DATA round (numData bits). The - // capture callback fires once per enqueue in chronological order, so the - // per-shot enqueue index distinguishes the heterogeneous final round. - g_enqueues_per_shot = static_cast(numRounds) + 1; - g_syndrome_bits_per_round = static_cast(numSyndromesPerRound); - g_data_bits = static_cast(numData); - g_syndrome_count = 0; - - printf("Syndrome capture enabled: saving to %s\n", - syndrome_filename.c_str()); - printf( - "Will capture %d enqueues per shot (%ld syndrome rounds of %d bits + " - "1 data round of %d bits)\n", - g_enqueues_per_shot, numRounds, g_syndrome_bits_per_round, g_data_bits); - - // Write metadata to file header - g_syndrome_output_file << "NUM_DATA " << numData << "\n"; - g_syndrome_output_file << "NUM_LOGICAL " << numLogical << "\n"; - g_syndrome_output_file.flush(); - - // Register capture callback with decoder library - cudaq::qec::decoding::host::_set_syndrome_capture_callback( - [](const uint8_t *data, size_t len) { - std::lock_guard lock(g_syndrome_file_mutex); - if (!g_syndrome_output_file.is_open()) - return; - - // Position of this enqueue within the current shot. - int enqueue_idx = g_syndrome_count % g_enqueues_per_shot; - - // Write shot boundary marker at the start of each shot - if (enqueue_idx == 0) { - int shot_num = g_syndrome_count / g_enqueues_per_shot; - g_syndrome_output_file << "SHOT_START " << shot_num << "\n"; - } - - // The last enqueue of each shot is the final DATA round (numData - // bits); all others are uniform syndrome rounds. Record the true bit - // width on the ROUND_START line so replay can chunk it back exactly - // -- truncating the data round to the syndrome width was the replay - // bug. - int bits_this_round = (enqueue_idx == g_enqueues_per_shot - 1) - ? g_data_bits - : g_syndrome_bits_per_round; - - g_syndrome_output_file << "ROUND_START " << g_syndrome_count << " " - << bits_this_round << "\n"; - - // Unpack syndrome data - each byte contains 8 bits (packed format, - // MSB first). Emit exactly bits_this_round bits. - int bits_written = 0; - for (size_t i = 0; i < len && bits_written < bits_this_round; i++) { - uint8_t byte = data[i]; - for (int bit_idx = 7; bit_idx >= 0; bit_idx--) { - if (bits_written >= bits_this_round) - break; - int bit = (byte >> bit_idx) & 1; - g_syndrome_output_file << bit << "\n"; - bits_written++; - } - } - g_syndrome_output_file.flush(); - - g_syndrome_count++; - }); - - // Set RNG seed for deterministic results - cudaq::set_random_seed(42); - printf("Set RNG seed to 42 for deterministic syndrome generation\n"); - } - - // Either run quantum simulation OR replay syndromes from file - std::vector> run_result; - - if (load_syndrome) { - // Syndrome replay mode - if (syndrome_filename.empty()) - throw std::runtime_error("--load_syndrome requires a filename argument"); - - printf("\n=== Syndrome Replay Mode ===\n"); - printf("Loading syndromes from: %s\n", syndrome_filename.c_str()); - - std::ifstream syndrome_file(syndrome_filename); - if (!syndrome_file) - throw std::runtime_error("Could not open syndrome file: " + - syndrome_filename); - - // Read header and syndrome data. Each shot is stored as a list of rounds, - // and each round keeps its own bit count (the final DATA round has numData - // bits, the syndrome rounds have numSyndromesPerRound). The per-round width - // is read from the "ROUND_START " header that the capture - // writes, so replay reconstructs the heterogeneous stream exactly. - std::size_t file_numData = 0; - std::size_t file_numLogical = 0; - std::vector saved_corrections; - std::vector>> saved_shots; - std::string line; - - bool reading_syndromes = false; - bool saw_corrections_start = false; - bool saw_corrections_end = false; - while (std::getline(syndrome_file, line)) { - if (line.find("NUM_DATA") == 0) { - std::istringstream iss(line); - std::string tag; - iss >> tag >> file_numData; - } else if (line.find("NUM_LOGICAL") == 0) { - std::istringstream iss(line); - std::string tag; - iss >> tag >> file_numLogical; - } else if (line.find("CORRECTIONS_START") == 0) { - saw_corrections_start = true; - while (std::getline(syndrome_file, line)) { - if (line.find("CORRECTIONS_END") == 0) { - saw_corrections_end = true; - break; - } - uint8_t correction_bit = static_cast(std::stoi(line)); - saved_corrections.push_back(correction_bit); - } - printf("Read %zu saved corrections\n", saved_corrections.size()); - break; - } else if (line.find("SHOT_START") == 0) { - saved_shots.emplace_back(); - reading_syndromes = true; - } else if (line.find("ROUND_START") == 0) { - // Start a new round, read greedily until the next marker. The resulting - // per-shot round count and widths are validated against the known - // geometry after parsing (see the structural-completeness check below), - // which is what catches a truncated/corrupt capture. - if (!saved_shots.empty()) - saved_shots.back().emplace_back(); - continue; - } else if (reading_syndromes) { - try { - int bit = std::stoi(line); - saved_shots.back().back().push_back(static_cast(bit)); - } catch (...) { - break; - } - } - } - - printf("Read %zu shots with syndromes\n", saved_shots.size()); - - // Require a STRUCTURALLY COMPLETE capture so a truncated/corrupt file fails - // loudly instead of silently "passing" with no verification. The file is - // app-generated, so the realistic corruption is truncation; validate it - // against the known geometry (num_rounds, numSyndromesPerRound, numData). - if (saved_shots.empty()) - throw std::runtime_error( - "no shots parsed from syndrome file (empty or corrupt): " + - syndrome_filename); - if (!saw_corrections_start || !saw_corrections_end) - throw std::runtime_error( - "syndrome file is missing the CORRECTIONS_START/CORRECTIONS_END " - "footer (truncated or incomplete capture): " + - syndrome_filename); - if (file_numData != numData || file_numLogical != numLogical) - throw std::runtime_error( - "syndrome file parameters (numData=" + std::to_string(file_numData) + - ", numLogical=" + std::to_string(file_numLogical) + - ") do not match this run (numData=" + std::to_string(numData) + - ", numLogical=" + std::to_string(numLogical) + ")"); - if (saved_corrections.size() != saved_shots.size()) - throw std::runtime_error( - "syndrome file has " + std::to_string(saved_corrections.size()) + - " corrections for " + std::to_string(saved_shots.size()) + - " shots; expected exactly one correction per shot (truncated " - "capture)"); - // Each shot must be the full enqueue stream: num_rounds syndrome rounds of - // numSyndromesPerRound bits + one final data round of numData bits. - const std::size_t expectedRounds = static_cast(numRounds) + 1; - for (std::size_t s = 0; s < saved_shots.size(); ++s) { - const auto &rounds = saved_shots[s]; - if (rounds.size() != expectedRounds) - throw std::runtime_error( - "syndrome file shot " + std::to_string(s) + " has " + - std::to_string(rounds.size()) + " rounds; expected " + - std::to_string(expectedRounds) + - " (num_rounds+1); truncated or wrong-geometry capture"); - for (std::size_t r = 0; r < rounds.size(); ++r) { - const std::size_t expectedBits = (r + 1 == expectedRounds) - ? static_cast(numData) - : numSyndromesPerRound; - if (rounds[r].size() != expectedBits) - throw std::runtime_error( - "syndrome file shot " + std::to_string(s) + " round " + - std::to_string(r) + " has " + std::to_string(rounds[r].size()) + - " bits; expected " + std::to_string(expectedBits) + - " (truncated or corrupt capture)"); - } - } - - syndrome_file.close(); - - // Process saved syndromes through decoder - printf("Feeding %zu shots of saved syndromes to decoder...\n", - saved_shots.size()); - - int corrections_matched = 0; - int corrections_mismatched = 0; - - for (size_t shot_idx = 0; shot_idx < saved_shots.size(); shot_idx++) { - // Reset decoder for new shot - for (size_t logical_idx = 0; logical_idx < numLogical; logical_idx++) { - cudaq::qec::decoding::reset_decoder(logical_idx); - } - - // Feed syndromes to the decoder one captured round at a time, each at its - // recorded width: num_rounds uniform syndrome rounds - // (numSyndromesPerRound bits) followed by the final DATA round (numData - // bits). This mirrors the live enqueue cadence exactly, so the boundary - // detectors -- which XOR the final data bits against the last-round - // Z-ancillas -- see the same data round the live path enqueued. - const auto &shot_rounds = saved_shots[shot_idx]; - for (const auto &round_bits : shot_rounds) { - // Replay path: raw syndrome bits read from a saved file. Use the - // test-only `enqueue_syndromes_test` API since these bits have no - // measurement-event identity to preserve and the production - // `enqueue_syndromes(vector&)` is the wrong shape. - std::vector syndrome_round; - syndrome_round.reserve(round_bits.size()); - for (uint8_t b : round_bits) - syndrome_round.push_back(static_cast(b)); - - // Enqueue this round for all logical qubits - for (size_t logical_idx = 0; logical_idx < numLogical; logical_idx++) { - cudaq::qec::decoding::enqueue_syndromes_test(logical_idx, - syndrome_round); - } - } - - // Get logical corrections from decoder - uint8_t correction_bit = 0; - for (size_t logical_idx = 0; logical_idx < numLogical; logical_idx++) { - auto corrections = - cudaq::qec::decoding::get_corrections(logical_idx, 1, false); - if (!corrections.empty() && corrections[0]) { - correction_bit = 1; - } - } - - // Compare with saved correction if available - if (shot_idx < saved_corrections.size()) { - if (correction_bit == saved_corrections[shot_idx]) { - corrections_matched++; - } else { - corrections_mismatched++; - if (corrections_mismatched <= 10) { - printf(" Shot %zu: mismatch! Replayed=%u, Saved=%u\n", shot_idx, - correction_bit, saved_corrections[shot_idx]); - } - } - } - } - - printf("Replay complete: %zu shots processed\n", saved_shots.size()); - if (!saved_corrections.empty()) { - printf("Correction verification: %d matched, %d mismatched\n", - corrections_matched, corrections_mismatched); - if (corrections_mismatched != 0) - throw std::runtime_error( - "replay correction mismatch: " + - std::to_string(corrections_mismatched) + " of " + - std::to_string(corrections_matched + corrections_mismatched) + - " shots differ from the captured run"); - printf("SUCCESS: All corrections match!\n"); - } - return; - - } else { - // Normal quantum simulation mode - printf("\n=== Quantum Simulation Mode ===\n"); - - // If this is a remote platform (not local sim nor emulation), don't use the - // noise model. - run_result = - cudaq::get_platform().is_remote() - ? cudaq::run(numShots, cudaq::qec::qpu::demo_circuit_qpu, - /*allow_device_calls=*/true, prep, numData, numAncx, - numAncz, numRounds, numLogical, cnot_schedX_flat, - cnot_schedZ_flat, p_spam_per_patch, - /*apply_corrections=*/true) - : cudaq::run(numShots, noise, cudaq::qec::qpu::demo_circuit_qpu, - /*allow_device_calls=*/true, prep, numData, numAncx, - numAncz, numRounds, numLogical, cnot_schedX_flat, - cnot_schedZ_flat, p_spam_per_patch, - /*apply_corrections=*/true); - } - printf("Result size: %ld\n", run_result.size()); - std::vector> logical_results; - auto obs_matrix = code.get_observables_z(); - int num_non_zero_values = 0; - std::int64_t num_corrections = 0; - std::vector per_decoder_corrections(numLogical, 0); - std::vector per_decoder_errors(numLogical, 0); - for (int i = 0; i < run_result.size(); i++) { - const auto &shot = run_result[i]; - const std::uint64_t corrections_mask = shot[numLogical]; - logical_results.emplace_back(); - for (int j = 0; j < numLogical; j++) { - if ((corrections_mask >> j) & 1) { - num_corrections++; - per_decoder_corrections[j]++; - } - std::vector result_vec(numData); - for (int l = 0; l < numData; l++) { - result_vec[l] = (shot[j] & (1ull << l)) ? 1.0 : 0.0; - } - cudaqx::tensor result_tensor; - cudaq::qec::convert_vec_soft_to_tensor_hard(result_vec, result_tensor); - // Calculate the logical observable for each logical qubit - uint8_t logical_result = (obs_matrix.dot(result_tensor) % 2).at({0}); - logical_results.back().push_back(logical_result); - if (logical_result != 0) { - num_non_zero_values++; - per_decoder_errors[j]++; - } - } - } - printf("Number of non-zero values measured : %d\n", num_non_zero_values); - printf("Number of corrections decoder found: %ld\n", num_corrections); - for (std::size_t j = 0; j < numLogical; j++) - printf("decoder[%zu] (%s): corrections=%ld, logical_errors=%ld/%zu\n", j, - j < loaded_types.size() ? loaded_types[j].c_str() : "unknown", - per_decoder_corrections[j], per_decoder_errors[j], numShots); - - // Save corrections to file if syndrome capture was enabled - if (save_syndrome && g_syndrome_output_file.is_open()) { - // Disable callback to stop capturing - cudaq::qec::decoding::host::_set_syndrome_capture_callback(nullptr); - - // Save logical corrections for each shot (for verification during replay) - g_syndrome_output_file << "CORRECTIONS_START\n"; - for (size_t i = 0; i < logical_results.size(); i++) { - // For multi-logical, just save whether any correction was applied - uint8_t any_correction = run_result[i][numLogical] != 0 ? 1 : 0; - g_syndrome_output_file << static_cast(any_correction) << "\n"; - } - g_syndrome_output_file << "CORRECTIONS_END\n"; - g_syndrome_output_file.close(); - printf("Syndrome data saved to: %s\n", syndrome_filename.c_str()); - } -} - -void show_help() { - printf("Usage: qec-test4 [options]\n"); - printf("Options:\n"); - printf(" --distance Distance of the surface code. Default: 5\n"); - printf(" --num_shots Number of shots. Default: 10\n"); - printf( - " --p_spam SPAM probability. Range[0, 1]. Default: 0.01\n"); - printf(" --p_spam_per_patch Comma-separated SPAM probabilities; " - "one per logical patch, or one value replicated to all patches. " - "Overrides --p_spam.\n"); - printf(" --num_logical Number of logical qubits. Default: 1\n"); - printf(" --num_rounds Number of measurement rounds. Default: " - "distance\n"); - printf(" --decoder_type Comma-separated decoder types to write " - "when generating a config (with --save_dem). Entries: 'pymatching', " - "'nv-qldpc-decoder', 'trt_decoder'. One entry per logical patch " - "(patch i decodes with entry i), or a single entry replicated to " - "all patches. Default: pymatching\n"); - printf(" --onnx_path ONNX model path (required with " - "--decoder_type trt_decoder and --save_dem)\n"); - printf(" --save_dem Generate the DEM + decoder config and save to " - "a YAML file (generation phase).\n"); - printf( - " --yaml Run realtime decoding from a YAML config; the " - "decoder is read from the file. Do not combine with --decoder_type.\n"); - printf(" --load_dem Alias of --yaml.\n"); - printf(" --save_syndrome Save syndrome data to a file for later " - "replay.\n"); - printf(" --load_syndrome Load and replay syndrome data from a " - "file.\n"); - printf(" --use-relay-bp For nv-qldpc-decoder entries: select Relay BP " - "instead of the default BP + OSD block. Accepted and ignored with " - "--yaml (the YAML is authoritative).\n"); - printf(" --ising_bundle

Ising d/T/Z bundle directory " - "(H_csr.bin/O_csr.bin/priors.bin/metadata.txt plus D_sparse.txt; " - "generated locally, not shipped -- run without it to print the " - "generation recipe). With --save_dem --decoder_type trt_decoder the " - "config carries the Ising H/O/priors and an Ising-ordered D_sparse " - "over the cudaqx live buffer.\n"); - printf(" --help Show this help message\n"); -} - -int main(int argc, char **argv) { -#ifdef QEC_APP_EXTERNAL_DECODING_SERVER - realtime_channel_guard realtime_channel; -#endif - int num_shots = 10; - int distance = 5; - double p_spam = 0.01; - std::vector p_spam_per_patch; - int num_logical = 1; - int num_rounds = -1; // Will be set to distance if not specified - bool save_dem = false; - bool load_dem = false; - std::string dem_filename; - - // Decoder type selection. This is a GENERATION-phase knob (used with - // --save_dem); with --yaml the decoder is read from the file. - std::string decoder_type = "pymatching"; // Default - bool decoder_type_explicit = false; - bool yaml_mode = false; - std::string onnx_path; - // Optional Ising d/T/Z bundle dir (generate_test_data.py output + the - // generated D_sparse.txt). When set with --save_dem --decoder_type - // trt_decoder, the trt config carries the Ising H/O/priors (Ising - // detector order) and an Ising-ordered D_sparse over the cudaqx live buffer. - std::string ising_bundle; - - // Syndrome save/load options - bool save_syndrome = false; - bool load_syndrome = false; - std::string syndrome_filename; - bool use_relay_bp = false; - - // Parse the command line arguments. Value-taking flags read the next argv - // entry through require_value, which errors out (rather than reading past the - // end of argv) when a flag is given with no following value; numeric flags go - // through require_int/require_double, which reject malformed or out-of-range - // values instead of aborting on an uncaught stoi/stod exception. - int i; - auto require_value = [&](const char *flag) -> std::string { - if (i + 1 >= argc) { - printf("Error: %s requires a value.\n", flag); - std::exit(1); - } - return argv[++i]; - }; - auto require_int = [&](const char *flag) -> int { - const std::string v = require_value(flag); - try { - std::size_t pos = 0; - int r = std::stoi(v, &pos); - if (pos != v.size()) - throw std::invalid_argument(v); - return r; - } catch (const std::exception &) { - printf("Error: %s expects an integer, got '%s'.\n", flag, v.c_str()); - std::exit(1); - } - }; - auto require_double = [&](const char *flag) -> double { - const std::string v = require_value(flag); - try { - std::size_t pos = 0; - double r = std::stod(v, &pos); - if (pos != v.size()) - throw std::invalid_argument(v); - return r; - } catch (const std::exception &) { - printf("Error: %s expects a number, got '%s'.\n", flag, v.c_str()); - std::exit(1); - } - }; - auto require_double_list = [&](const char *flag) { - const std::string value = require_value(flag); - std::vector parsed_values; - std::stringstream stream(value); - std::string token; - while (std::getline(stream, token, ',')) { - try { - std::size_t pos = 0; - double parsed = std::stod(token, &pos); - if (token.empty() || pos != token.size()) - throw std::invalid_argument(token); - parsed_values.push_back(parsed); - } catch (const std::exception &) { - printf("Error: %s expects a comma-separated list of numbers, got " - "'%s'.\n", - flag, value.c_str()); - std::exit(1); - } - } - if (parsed_values.empty() || value.back() == ',') { - printf("Error: %s expects a non-empty comma-separated list.\n", flag); - std::exit(1); - } - return parsed_values; - }; - for (i = 1; i < argc; i++) { - std::string arg = argv[i]; - if (arg == "--distance") { - distance = require_int("--distance"); - } else if (arg == "--num_shots") { - num_shots = require_int("--num_shots"); - } else if (arg == "--p_spam") { - p_spam = require_double("--p_spam"); - } else if (arg == "--p_spam_per_patch" || arg == "--p-spam-per-patch") { - p_spam_per_patch = require_double_list("--p_spam_per_patch"); - } else if (arg == "--help" || arg == "-h") { - show_help(); - return 0; - } else if (arg == "--num_logical") { - num_logical = require_int("--num_logical"); - } else if (arg == "--num_rounds") { - num_rounds = require_int("--num_rounds"); - } else if (arg == "--decoder_type") { - decoder_type = require_value("--decoder_type"); - decoder_type_explicit = true; - } else if (arg == "--onnx_path" || arg == "--onnx-path") { - onnx_path = require_value("--onnx_path"); - } else if (arg == "--ising_bundle" || arg == "--ising-bundle") { - ising_bundle = require_value("--ising_bundle"); - } else if (arg == "--save_dem") { - save_dem = true; - dem_filename = require_value("--save_dem"); - } else if (arg == "--yaml" || arg == "--load_dem") { - // Realtime phase: the decoder is read from the YAML (authoritative). - load_dem = true; - yaml_mode = true; - dem_filename = require_value("--yaml"); - } else if (arg == "--save_syndrome") { - save_syndrome = true; - syndrome_filename = require_value("--save_syndrome"); - } else if (arg == "--load_syndrome") { - load_syndrome = true; - syndrome_filename = require_value("--load_syndrome"); - } else if (arg == "--use-relay-bp") { - use_relay_bp = true; - } else { - printf("Unknown argument: %s\n", arg.c_str()); - show_help(); - return 1; - } - } - - if (!load_dem && !save_dem && !load_syndrome) { - printf("Neither --save_dem nor --load_dem nor --load_syndrome was " - "specified. This is not a valid use case for this program.\n"); - show_help(); - return 1; - } - - // Validate syndrome save/load options - if (save_syndrome && load_syndrome) { - printf("Error: Cannot use both --save_syndrome and --load_syndrome " - "together\n"); - return 1; - } - if (save_syndrome && save_dem) { - printf("Error: Cannot use --save_syndrome with --save_dem\n"); - printf(" --save_dem returns early without running simulation.\n"); - return 1; - } - // Syndrome capture/replay records one stream per shot; the multi-logical - // packing is not handled, so restrict it to a single logical qubit. - if ((save_syndrome || load_syndrome) && num_logical != 1) { - printf("Error: --save_syndrome/--load_syndrome support num_logical=1 " - "only.\n"); - return 1; - } - - // Reject geometries the rotated surface-code kernel cannot build (it aborts - // inside the kernel otherwise): the distance must be an odd integer >= 3. - if (distance < 3 || distance % 2 == 0) { - printf("Error: distance must be an odd integer >= 3 (got %d).\n", distance); - return 1; - } - // Each patch's final data bits are returned in one 64-bit word, so numData - // = d^2 must fit (d <= 7). Larger distances need the word split across - // multiple result entries, which this example does not implement. Widened - // multiply: d^2 overflows int from d = 46341. - const std::int64_t num_data_bits = - static_cast(distance) * distance; - if (num_data_bits >= 64) { - printf("Error: distance %d has %lld data qubits; only numData < 64 " - "(d <= 7) is supported.\n", - distance, static_cast(num_data_bits)); - return 1; - } - // Per-patch correction bits are returned in one 64-bit mask (example - // policy; the HOST realtime path itself imposes no decoder-count cap). - // Validate BEFORE the decoder-type list is replicated num_logical times. - if (num_logical < 1 || num_logical > 64) { - printf("Error: --num_logical must be in [1, 64] (got %d).\n", num_logical); - return 1; - } - if (num_shots < 1) { - printf("Error: --num_shots must be >= 1 (got %d).\n", num_shots); - return 1; - } - if (!std::isfinite(p_spam) || p_spam < 0.0 || p_spam > 1.0) { - printf("Error: --p_spam must be a finite value in [0, 1] (got %g).\n", - p_spam); - return 1; - } - if (p_spam_per_patch.empty()) - p_spam_per_patch.assign(num_logical, p_spam); - else if (p_spam_per_patch.size() == 1 && num_logical > 1) - p_spam_per_patch.assign(num_logical, p_spam_per_patch.front()); - else if (p_spam_per_patch.size() != static_cast(num_logical)) { - printf("Error: --p_spam_per_patch lists %zu values; expected 1 or " - "num_logical (%d).\n", - p_spam_per_patch.size(), num_logical); - return 1; - } - for (std::size_t patch = 0; patch < p_spam_per_patch.size(); ++patch) { - const double value = p_spam_per_patch[patch]; - if (!std::isfinite(value) || value < 0.0 || value > 1.0) { - printf("Error: --p_spam_per_patch[%zu] must be a finite value in [0, " - "1] (got %g).\n", - patch, value); - return 1; - } - } - - // Set defaults if not specified - if (num_rounds == -1) - num_rounds = distance; - // --yaml is authoritative for the decoder; a co-passed --decoder_type would - // be ambiguous, so reject the combination. - if (yaml_mode && decoder_type_explicit) { - printf( - "Error: --decoder_type only applies to --save_dem (generation). " - "With --yaml the decoder is read from the file; do not pass both.\n"); - return 1; - } - // --save_dem (generation) and --yaml (realtime) are separate phases that - // write and read the same config file. Passing both silently drops the save, - // so reject the combination rather than surprise the user. - if (yaml_mode && save_dem) { - printf("Error: --save_dem (generation) and --yaml (realtime) are separate " - "phases; do not pass both.\n"); - return 1; - } - - // Split --decoder_type into the per-patch type list and validate every - // entry. A single entry is replicated to all patches (legacy behavior); a - // list must name exactly one decoder per patch. - std::vector decoder_types; - { - std::stringstream ss(decoder_type); - std::string tok; - while (std::getline(ss, tok, ',')) - decoder_types.push_back(tok); - if (decoder_type.empty() || decoder_type.back() == ',') - decoder_types.push_back(std::string()); - } - for (const auto &t : decoder_types) { - bool supported = - t == "pymatching" || t == "nv-qldpc-decoder" || t == "trt_decoder"; -#ifdef QEC_APP_EXTERNAL_DECODING_SERVER - supported = supported || t == "concurrency_test_decoder"; -#endif - if (!supported) { - printf("Error: --decoder_type entries must be 'pymatching', " - "'nv-qldpc-decoder', or 'trt_decoder' (got '%s')\n", - t.c_str()); - return 1; - } - } - if (save_dem && decoder_types.size() != 1 && - decoder_types.size() != static_cast(num_logical)) { - printf("Error: --decoder_type lists %zu entries; expected 1 or " - "num_logical (%d)\n", - decoder_types.size(), num_logical); - return 1; - } - if (decoder_types.size() == 1 && num_logical > 1) - decoder_types.assign(num_logical, decoder_types[0]); - auto has_type = [&](const char *t) { - return std::find(decoder_types.begin(), decoder_types.end(), t) != - decoder_types.end(); - }; - - if (save_dem && has_type("trt_decoder") && onnx_path.empty()) { - printf("Error: --onnx_path is required with a trt_decoder entry and " - "--save_dem\n"); - return 1; - } - if (save_dem && !onnx_path.empty() && !has_type("trt_decoder")) - printf("Warning: --onnx_path is only used by trt_decoder entries; " - "ignoring it.\n"); - - // --use-relay-bp configures nv-qldpc-decoder entries at generation. With - // --yaml it is accepted and ignored (the YAML is authoritative; the test - // driver passes the same extra args to both phases). - if (save_dem && use_relay_bp && !has_type("nv-qldpc-decoder")) { - printf("Error: --use-relay-bp requires an 'nv-qldpc-decoder' entry in " - "--decoder_type.\n"); - return 1; - } - - // The example decodes ONE volume of num_rounds rounds (no sliding windows), - // so there is no window-divisibility constraint: any num_rounds >= 2 is a - // representable memory experiment (e.g. d5/T6). Require >= 2 because a single - // round has no cross-round detectors (pairedRounds = num_rounds - 1 = 0) and - // therefore no temporal error information -- not a meaningful memory run. - // num_rounds < distance is decodable but not fault-tolerant, so warn rather - // than reject -- the API should still express it. - if (num_rounds < 2) { - printf("Error: num_rounds (%d) must be >= 2 (a memory experiment needs at " - "least one cross-round detector).\n", - num_rounds); - return 1; - } - if (num_rounds < distance) - printf("Warning: num_rounds (%d) < distance (%d): decodable but not " - "fault-tolerant (fewer rounds than the code distance).\n", - num_rounds, distance); - - // Syndrome replay feeds saved syndromes through a configured decoder, which - // only exists under --yaml. Replaying without --yaml would call - // reset_decoder() on an unconfigured decoder ("Decoder 0 not found"), so - // require it. - if (load_syndrome && !yaml_mode) { - printf("Error: --load_syndrome requires --yaml (the decoder to replay " - "through is configured from the loaded config).\n"); - return 1; - } - - // --ising_bundle is only consumed by the trt+Ising generation path - // (--save_dem --decoder_type trt_decoder). Warn rather than silently ignore - // it on any other path so a stray bundle argument is visible. - if (!ising_bundle.empty() && !(save_dem && has_type("trt_decoder"))) { - printf("Warning: --ising_bundle is only used with --save_dem and a " - "trt_decoder entry; ignoring it on this path.\n"); - } - - // The trt+Ising path needs an external predecoder bundle that is generated - // locally and not shipped with this repository. If the bundle is absent (no - // metadata.txt), stop with the exact generation recipe rather than failing - // deeper in. - if (save_dem && has_type("trt_decoder") && !ising_bundle.empty()) { - if (!std::ifstream(ising_bundle + "/metadata.txt")) { - printf( - "This example's trt+Ising path requires the Ising predecoder bundle, " - "which is generated locally and not shipped in this repository.\n" - " '%s/metadata.txt' was not found.\n" - "Generate it from the Ising decoding project " - "(https://github.com/NVIDIA/Ising-Decoding) into '%s':\n" - " 1. python generate_test_data.py --distance %d --n-rounds %d " - "--basis Z --code-rotation XV --output-dir %s\n" - " 2. surface_code-4-yaml --save_dem cfg.yml --decoder_type " - "pymatching --distance %d --num_rounds %d > sched.txt\n" - " python gen_dsparse_from_memory_circuit.py %d %d Z XV sched.txt " - "%s/D_sparse.txt --ising-repo /path/to/ising/code\n" - " 3. export the ONNX predecoder predecoder_memory_d%d_T%d_Z.onnx " - "and " - "pass it via --onnx_path.\n" - "Then re-run with --ising_bundle %s.\n", - ising_bundle.c_str(), ising_bundle.c_str(), distance, num_rounds, - ising_bundle.c_str(), distance, num_rounds, distance, num_rounds, - ising_bundle.c_str(), distance, num_rounds, ising_bundle.c_str()); - return 1; - } - } - - printf("Running with p_spam_per_patch = ["); - for (std::size_t patch = 0; patch < p_spam_per_patch.size(); ++patch) - printf("%s%g", patch ? ", " : "", p_spam_per_patch[patch]); - printf("], distance = %d, num_shots = %d, num_rounds = %d\n", distance, - num_shots, num_rounds); - - // Build the code at code_rotation XV (the predecoder's training orientation), - // which sets the geometry and observable basis. The DEM-generation kernel - // (dem_gen_circuit) emits the full detector structure -- prep singles + - // data-derived boundary detectors, X-then-Z order. - try { -#ifdef QEC_APP_EXTERNAL_DECODING_SERVER - if (load_dem) - realtime_channel.initialize(argv[0]); -#endif - - auto code = cudaq::qec::get_code( - "surface_code", - cudaqx::heterogeneous_map{{"distance", distance}, - {"orientation", std::string("XV")}}); - - demo_circuit_host(*code, distance, p_spam_per_patch, - cudaq::qec::operation::prep0, num_shots, num_rounds, - num_logical, dem_filename, save_dem, load_dem, - decoder_types, save_syndrome, load_syndrome, - syndrome_filename, use_relay_bp, onnx_path, ising_bundle); - } catch (const std::exception &e) { - // Configuration, channel, and geometry failures surface as a clean error - // rather than an uncaught-exception abort. - printf("Error: %s\n", e.what()); - cudaq::qec::decoding::config::finalize_decoders(); - return 1; - } - - // Ensure clean shutdown - cudaq::qec::decoding::config::finalize_decoders(); - - return 0; -} diff --git a/libs/qec/unittests/realtime/app_examples/surface_code_1.py b/libs/qec/unittests/realtime/app_examples/surface_code_1.py index 7b23f9f82..e7168e165 100644 --- a/libs/qec/unittests/realtime/app_examples/surface_code_1.py +++ b/libs/qec/unittests/realtime/app_examples/surface_code_1.py @@ -57,7 +57,9 @@ def save_dem_to_file(dem, dem_filename, numSyndromesPerRound, num_logical): config.O_sparse = qec.pcm_to_sparse_vec(dem.observables_flips_matrix) config.D_sparse = qec.generate_timelike_sparse_detector_matrix( numSyndromesPerRound, numRounds, False) - config.decoder_custom_args = {"lut_error_depth": 2} + multi_error_lut_config = qec.multi_error_lut_config() + multi_error_lut_config.lut_error_depth = 2 + config.set_decoder_custom_args(multi_error_lut_config) decoders.append(config) multi_config.decoders = decoders diff --git a/libs/qec/unittests/realtime/mock_decode_setup.h b/libs/qec/unittests/realtime/mock_decode_setup.h index 333f26361..4e2b37222 100644 --- a/libs/qec/unittests/realtime/mock_decode_setup.h +++ b/libs/qec/unittests/realtime/mock_decode_setup.h @@ -122,8 +122,6 @@ inline std::vector load_syndromes(const std::string &path, } if (line.find("NUM_DATA") == 0 || line.find("NUM_LOGICAL") == 0) { continue; - } else if (line.find("ROUND_START") == 0) { - continue; } else if (reading_shot) { line.erase(0, line.find_first_not_of(" \t\n\r")); line.erase(line.find_last_not_of(" \t\n\r") + 1); diff --git a/libs/qec/unittests/realtime/qec_graph_decode_test/qldpc_config_loader.cpp b/libs/qec/unittests/realtime/qec_graph_decode_test/qldpc_config_loader.cpp deleted file mode 100644 index 921f812ed..000000000 --- a/libs/qec/unittests/realtime/qec_graph_decode_test/qldpc_config_loader.cpp +++ /dev/null @@ -1,95 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2025-2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#include "qldpc_config_loader.h" - -#include "cudaq/qec/decoder.h" -#include "cudaq/qec/realtime/decoding_config.h" -#include "cudaq/qec/realtime/sparse_to_csr.h" - -#include -#include -#include -#include -#include - -namespace test_realtime_qldpc { - -namespace { - -std::string read_file(const std::string &path) { - std::ifstream f(path); - if (!f.is_open()) - throw std::runtime_error( - "test_realtime_qldpc_config_loader: failed to open YAML: " + path); - return std::string((std::istreambuf_iterator(f)), - std::istreambuf_iterator()); -} - -} // namespace - -LoadedDecoder load_decoder_from_yaml(const std::string &yaml_path) { - using namespace cudaq::qec; - - auto mdc = decoding::config::multi_decoder_config::from_yaml_str( - read_file(yaml_path)); - if (mdc.decoders.size() != 1u) - throw std::runtime_error( - "test_realtime_qldpc_config_loader: expected exactly one decoder in " - "YAML, found " + - std::to_string(mdc.decoders.size()) + ": " + yaml_path); - - auto &dec = mdc.decoders[0]; - - // H_sparse -> dense tensor (the plugin reads it through the standard - // decoder::get() entry point). - std::vector h_row_ptr, h_col_idx; - std::size_t h_rows = - realtime::sparse_vec_to_csr(dec.H_sparse, h_row_ptr, h_col_idx); - if (h_rows != dec.syndrome_size) - throw std::runtime_error( - "test_realtime_qldpc_config_loader: H_sparse row count " + - std::to_string(h_rows) + " does not match dec.syndrome_size " + - std::to_string(dec.syndrome_size)); - - const std::size_t bs = dec.block_size; - const std::size_t ss = dec.syndrome_size; - cudaqx::tensor H_tensor({ss, bs}); - for (std::size_t r = 0; r < ss; ++r) - for (std::uint32_t j = h_row_ptr[r]; j < h_row_ptr[r + 1]; ++j) - H_tensor.at({r, static_cast(h_col_idx[j])}) = 1; - - auto params = dec.decoder_custom_args_to_heterogeneous_map(); - auto plugin = decoder::get("nv-qldpc-decoder", H_tensor, params); - if (!plugin) - throw std::runtime_error( - "test_realtime_qldpc_config_loader: decoder::get(\"nv-qldpc-decoder\"," - " ...) returned nullptr; is the plugin built and discoverable?"); - plugin->set_D_sparse(dec.D_sparse); - plugin->set_O_sparse(dec.O_sparse); - - LoadedDecoder out{}; - out.decoder = std::move(plugin); - - // num_measurements is the highest column index referenced by D_sparse, +1. - // (D_sparse columns are measurement-bit indices; CSR sparse_vec_to_csr - // gives us them via `d_ci`.) - std::vector d_rp, d_ci; - realtime::sparse_vec_to_csr(dec.D_sparse, d_rp, d_ci); - out.num_measurements = 0; - for (auto c : d_ci) - out.num_measurements = - std::max(out.num_measurements, static_cast(c + 1)); - - std::vector o_rp, o_ci; - out.num_observables = realtime::sparse_vec_to_csr(dec.O_sparse, o_rp, o_ci); - - return out; -} - -} // namespace test_realtime_qldpc diff --git a/libs/qec/unittests/realtime/qec_graph_decode_test/qldpc_config_loader.h b/libs/qec/unittests/realtime/qec_graph_decode_test/qldpc_config_loader.h deleted file mode 100644 index 781741526..000000000 --- a/libs/qec/unittests/realtime/qec_graph_decode_test/qldpc_config_loader.h +++ /dev/null @@ -1,45 +0,0 @@ -/****************************************************************-*- C++ -*-**** - * Copyright (c) 2025-2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -#pragma once - -// Tiny shim that does the YAML -> decoder construction in a .cpp TU so the -// .cu test file doesn't have to include cudaq/qec/realtime/decoding_config.h. -// That header pulls in C++20-only `bool operator==(...) const = default;` -// patterns that nvcc 13 chokes on when forced to C++20 (libstdc++ 13 ICE). -// -// The test's .cu file calls `load_decoder_from_yaml` to get back exactly the -// state it needs to bring up the dispatchers and replay syndromes (decoder, -// H_tensor's shape via num_measurements / num_observables, decoder_id is -// always 0 for this single-decoder test). - -#include -#include -#include - -namespace cudaq::qec { -class decoder; -} - -namespace test_realtime_qldpc { - -struct LoadedDecoder { - std::unique_ptr decoder; - std::size_t num_measurements = 0; ///< total per-shot, derived from D_sparse - std::size_t num_observables = 0; ///< rows of O_sparse -}; - -/// Read the YAML file at `yaml_path`, parse it via -/// `cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str`, build -/// the H_tensor, instantiate the nv-qldpc-decoder plugin, and feed it the -/// D_sparse + O_sparse from the YAML. Returns the constructed decoder plus -/// derived shape information. Aborts via std::runtime_error on YAML/decoder -/// errors so the gtest body can ASSERT_NO_THROW around the call. -LoadedDecoder load_decoder_from_yaml(const std::string &yaml_path); - -} // namespace test_realtime_qldpc diff --git a/libs/qec/unittests/realtime/qec_graph_decode_test/test_realtime_qldpc_graph_decoding.cpp b/libs/qec/unittests/realtime/qec_graph_decode_test/test_realtime_qldpc_graph_decoding.cpp index 13c1d0b26..713d3e2a1 100644 --- a/libs/qec/unittests/realtime/qec_graph_decode_test/test_realtime_qldpc_graph_decoding.cpp +++ b/libs/qec/unittests/realtime/qec_graph_decode_test/test_realtime_qldpc_graph_decoding.cpp @@ -1,81 +1,47 @@ /****************************************************************-*- C++ -*-**** * Copyright (c) 2025-2026 NVIDIA Corporation & Affiliates. * * All rights reserved. * + * * * This source code and the accompanying materials are made available under * * the terms of the Apache License 2.0 which accompanies this distribution. * ******************************************************************************/ /// @file test_realtime_qldpc_graph_decoding.cpp -/// @brief CI test for the per-round CUDA-graph relay BP decode trio, -/// exercising the full shared-ring two-dispatcher path -/// (CUDAQ_DISPATCH_PATH_HOST for per-round enqueue + CUDAQ_DISPATCH_PATH_DEVICE -/// for get_corrections and reset_decoder). Post-Step-8 of the realtime-session -/// refactor, this test consumes `qec_realtime_session` (for ring/dispatcher -/// setup) and `rpc_producer` (for host-side enqueue / get_corrections / reset -/// RPC calls), which are the exact same components surface_code-1-local uses in -/// its CUDAQ_QEC_REALTIME_MODE=inproc_rpc path -- so this test is now a -/// wire-format-and-orchestration-faithful contract test of the production -/// stack rather than a parallel re-implementation. +/// @brief CI test for the CPU-launched CUDA graph relay BP decode path, +/// exercising the full libcudaq-realtime CUDAQ_DISPATCH_PATH_HOST dispatch. /// /// Flow: -/// 1. Loads the relay BP config YAML and the syndrome fixture -/// (post-ROUND_START markers). -/// 2. Creates the nv-qldpc decoder via the .cpp shim around decoder::get() -/// and stores it in a single-element vector owned by the fixture (the -/// session takes a reference to that vector). -/// 3. Constructs a `qec_realtime_session` over that vector + the -/// libcudaq-realtime-dispatch.a launch fn (passed in from the exe -/// because the .a is hidden-visibility and the .so can't reference it -/// directly). `session.initialize()` then: -/// - calls capture_decode_graph() per decoder (each captured graph -/// publishes the canonical kEnqueueSyndromesFunctionId; the -/// session sub-routes them by routing_key = decoder_id per -/// proposals/cudaq_realtime_host_api.bs#host-path-graph-routing-key), -/// - allocates the TWO-RING data backing (rx_data != tx_data) + -/// flags per proposals/decoder_server_runtime.md, -/// - builds the N+2 function table (N GRAPH_LAUNCH sharing -/// function_id + distinct routing_key, 2 DEVICE_CALL), -/// - starts HOST_LOOP (CPU thread) + DEVICE_LOOP (persistent GPU -/// cooperative kernel). -/// 4. For each shot: replays per-round measurements via N -/// rpc_producer::enqueue_syndromes calls (each returns an empty -/// 24-byte RPCResponse, per the spec's Always-Emitted RPCResponse -/// rule -- no did_decode flag on the wire), then one -/// rpc_producer::get_corrections (asserts first byte matches -/// SyndromeEntry::expected_correction). -/// 5. After the loop: one rpc_producer::reset_decoder, then one more -/// rpc_producer::get_corrections, asserting the corrections buffer is -/// now zero. +/// 1. Loads a relay BP config YAML and syndrome data +/// 2. Creates the decoder via the generic decoder::get() API +/// 3. Calls capture_decode_graph() to get an opaque graph_resources* +/// 4. Wires the libcudaq-realtime C API: manager -> dispatcher (HOST_LOOP) +/// -> ringbuffer -> function table (GRAPH_LAUNCH) -> mailbox -> start +/// 5. For each syndrome: writes an RPC request into a ring buffer slot, +/// signals the slot, the host dispatcher launches the CUDA graph, +/// and the test polls for the RPCResponse and verifies corrections. #include #include -#include #include #include #include #include #include #include -#include #include #include +#include #include #include "cudaq/qec/decoder.h" +#include "cudaq/qec/realtime/decoding_config.h" +#include "cudaq/qec/realtime/graph_resources.h" +#include "cudaq/qec/realtime/sparse_to_csr.h" #include "cudaq/realtime/daemon/dispatcher/cudaq_realtime.h" #include "cudaq/realtime/daemon/dispatcher/dispatch_kernel_launch.h" -#include "qec_realtime_session.h" -#include "rpc_producer.h" -#include "cudaq/qec/realtime/decoder_rpc_ids.h" - -// YAML -> decoder construction lives in a .cpp shim so this .cu file doesn't -// need to include decoding_config.h, which uses C++20 `= default` operator== -// patterns that nvcc 13 ICEs on under -std=c++20 + libstdc++ 13. -#include "qldpc_config_loader.h" - #ifndef TEST_DATA_DIR #define TEST_DATA_DIR "." #endif @@ -90,19 +56,11 @@ using namespace cudaq::qec; using namespace cudaq::realtime; //============================================================================== -// Syndrome file loader (unchanged from pre-Step-8 -- this only touches the -// fixture format, not the dispatcher contract). +// Syndrome file loader //============================================================================== struct SyndromeEntry { - // Flat per-shot bits (all rounds concatenated). Kept for backward - // compatibility with the pre-migration test body. Total size equals - // num_measurements when the shot is complete. std::vector measurements; - // Authoritative per-round slices, populated from ROUND_START markers. - // Each inner vector is one round's worth of measurement bytes - // (measurements_per_round bytes for nv-qldpc). num_rounds = inner size. - std::vector> per_round_measurements; uint8_t expected_correction; }; @@ -116,26 +74,6 @@ static std::vector load_syndromes(const std::string &path, std::string line; bool in_corrections = false; std::size_t correction_idx = 0; - // Per-round slicing state, valid only while not in CORRECTIONS_START block. - // saw_round_start_in_current_shot lets us assert that every SHOT_START - // block (post-migration) contains at least one ROUND_START before either - // the next SHOT_START or CORRECTIONS_START -- per the recorder-round-markers - // todo "fail-fast (assertion) if any shot lacks ROUND_START markers". - bool saw_round_start_in_current_shot = false; - - auto seal_current_shot = [&]() { - if (entries.empty()) - return; - if (!saw_round_start_in_current_shot) { - throw std::runtime_error( - "test_realtime_qldpc_graph_decoding: load_syndromes: shot " + - std::to_string(entries.size() - 1) + - " is missing ROUND_START markers. Re-record the fixture (see " - "re-record-relay-fixture todo) -- pre-marker fixtures cannot be " - "consumed by this loader because per-round slicing is mandatory " - "for the migrated dispatcher-contract test."); - } - }; while (std::getline(file, line)) { if (line.empty()) @@ -143,7 +81,6 @@ static std::vector load_syndromes(const std::string &path, if (line.rfind("NUM_DATA", 0) == 0 || line.rfind("NUM_LOGICAL", 0) == 0) continue; if (line.rfind("CORRECTIONS_START", 0) == 0) { - seal_current_shot(); in_corrections = true; correction_idx = 0; continue; @@ -152,23 +89,9 @@ static std::vector load_syndromes(const std::string &path, break; if (line.rfind("SHOT_START", 0) == 0) { - seal_current_shot(); entries.emplace_back(); entries.back().measurements.reserve(num_measurements); entries.back().expected_correction = 0; - saw_round_start_in_current_shot = false; - continue; - } - - if (line.rfind("ROUND_START", 0) == 0) { - if (entries.empty()) { - throw std::runtime_error( - "test_realtime_qldpc_graph_decoding: load_syndromes: " - "ROUND_START before any SHOT_START in fixture: " + - path); - } - entries.back().per_round_measurements.emplace_back(); - saw_round_start_in_current_shot = true; continue; } @@ -178,299 +101,338 @@ static std::vector load_syndromes(const std::string &path, static_cast(std::stoi(line)); correction_idx++; } else if (!entries.empty()) { - uint8_t bit = static_cast(std::stoi(line)); - entries.back().measurements.push_back(bit); - if (saw_round_start_in_current_shot) - entries.back().per_round_measurements.back().push_back(bit); + entries.back().measurements.push_back( + static_cast(std::stoi(line))); } } - if (!in_corrections) - seal_current_shot(); return entries; } +//============================================================================== +// Ring buffer helpers +//============================================================================== + +static bool allocate_ring_buffer(std::size_t num_slots, std::size_t slot_size, + volatile uint64_t **host_flags_out, + volatile uint64_t **device_flags_out, + uint8_t **host_data_out, + uint8_t **device_data_out) { + void *hf = nullptr; + if (cudaHostAlloc(&hf, num_slots * sizeof(uint64_t), cudaHostAllocMapped) != + cudaSuccess) + return false; + void *df = nullptr; + if (cudaHostGetDevicePointer(&df, hf, 0) != cudaSuccess) { + cudaFreeHost(hf); + return false; + } + void *hd = nullptr; + if (cudaHostAlloc(&hd, num_slots * slot_size, cudaHostAllocMapped) != + cudaSuccess) { + cudaFreeHost(hf); + return false; + } + void *dd = nullptr; + if (cudaHostGetDevicePointer(&dd, hd, 0) != cudaSuccess) { + cudaFreeHost(hf); + cudaFreeHost(hd); + return false; + } + memset(hf, 0, num_slots * sizeof(uint64_t)); + *host_flags_out = static_cast(hf); + *device_flags_out = static_cast(df); + *host_data_out = static_cast(hd); + *device_data_out = static_cast(dd); + return true; +} + +static void free_ring_buffer(volatile uint64_t *host_flags, + uint8_t *host_data) { + if (host_flags) + cudaFreeHost(const_cast(host_flags)); + if (host_data) + cudaFreeHost(host_data); +} + //============================================================================== // GTest fixture //============================================================================== class GraphDecodeTest : public ::testing::Test { protected: - // The decoder vector that backs the session. Single-element today; the - // session is constructed with a reference to this so the test (and the - // session) share ownership semantics with the production path's - // `g_decoders` global. - std::vector> decoders_; - - // The realtime session abstracts: shared ring buffer, function table, - // HOST_LOOP CPU dispatcher (for per-round GRAPH_LAUNCH enqueue), and - // DEVICE_LOOP persistent GPU dispatcher (for DEVICE_CALL get_corrections - // and reset_decoder). Lifetime managed by SetUp/TearDown so each test - // method gets a clean dispatcher pair. - std::unique_ptr session_; + std::unique_ptr decoder_; + realtime::graph_resources *graph_res_ = nullptr; std::vector syndromes_; std::size_t num_measurements_ = 0; std::size_t num_observables_ = 0; - std::size_t num_rounds_ = 0; - std::size_t measurements_per_round_ = 0; - // decoder_id chosen by the plugin at capture_decode_graph() time; lives - // in the [0, kNvQldpcMaxDecoders) range. Cached so we can echo it into - // every RPC payload (enqueue / get_corrections / reset). Session - // initialize() walks decoders_ from index 0; this test creates exactly - // one decoder, so decoder_id_ is 0. - std::size_t decoder_id_ = 0; + + static constexpr std::size_t num_slots_ = 4; + std::size_t slot_size_ = 0; + + volatile uint64_t *rx_flags_host_ = nullptr; + volatile uint64_t *rx_flags_dev_ = nullptr; + uint8_t *rx_data_host_ = nullptr; + uint8_t *rx_data_dev_ = nullptr; + volatile uint64_t *tx_flags_host_ = nullptr; + volatile uint64_t *tx_flags_dev_ = nullptr; + uint8_t *tx_data_host_ = nullptr; + uint8_t *tx_data_dev_ = nullptr; + + cudaq_ringbuffer_t ringbuffer_{}; + cudaq_dispatch_manager_t *manager_ = nullptr; + cudaq_dispatcher_t *dispatcher_ = nullptr; + cudaq_function_entry_t host_table_[1]{}; + int shutdown_flag_ = 0; + uint64_t stats_counter_ = 0; void SetUp() override { int device_count = 0; cudaGetDeviceCount(&device_count); if (device_count == 0) GTEST_SKIP() << "No CUDA devices available"; - - // The self-relaunching device-graph scheduler this test drives uses - // device-side graph launch, which requires compute capability 9.0+ - // (Hopper). Skip on older GPUs (e.g. A100/sm_80 CI runners), matching the - // cuda-quantum dispatch-kernel tests. - int device = 0; - cudaGetDevice(&device); - cudaDeviceProp prop{}; - cudaGetDeviceProperties(&prop, device); - if (prop.major < 9) - GTEST_SKIP() << "Graph device launch requires compute capability 9.0+, " - "found " - << prop.major << "." << prop.minor; - cudaError_t flags_err = cudaSetDeviceFlags(cudaDeviceMapHost); ASSERT_TRUE(flags_err == cudaSuccess || flags_err == cudaErrorSetOnActiveProcess); - // ---- Load config + build decoder via the .cpp shim ---- - test_realtime_qldpc::LoadedDecoder loaded; - try { - loaded = test_realtime_qldpc::load_decoder_from_yaml( - std::string(TEST_DATA_DIR) + "/config_nv_qldpc_relay.yml"); - } catch (const std::exception &e) { - FAIL() << "load_decoder_from_yaml threw: " << e.what(); - } - num_measurements_ = loaded.num_measurements; - num_observables_ = loaded.num_observables; - ASSERT_NE(loaded.decoder, nullptr); - printf("Config: num_measurements=%zu, num_observables=%zu\n", - num_measurements_, num_observables_); - - decoders_.clear(); - decoders_.push_back(std::move(loaded.decoder)); - - // ---- Load syndromes (post-ROUND_START fixture) ---- + // --- Load config via public API --- + auto mdc = decoding::config::multi_decoder_config::from_yaml_str( + read_file(std::string(TEST_DATA_DIR) + "/config_nv_qldpc_relay.yml")); + ASSERT_EQ(mdc.decoders.size(), 1u); + auto &dec = mdc.decoders[0]; + + std::vector h_row_ptr, h_col_idx; + std::size_t h_rows = + realtime::sparse_vec_to_csr(dec.H_sparse, h_row_ptr, h_col_idx); + ASSERT_EQ(h_rows, dec.syndrome_size); + + std::size_t bs = dec.block_size; + std::size_t ss = dec.syndrome_size; + cudaqx::tensor H_tensor({ss, bs}); + for (std::size_t r = 0; r < ss; ++r) + for (uint32_t j = h_row_ptr[r]; j < h_row_ptr[r + 1]; ++j) + H_tensor.at({r, static_cast(h_col_idx[j])}) = 1; + + auto params = dec.decoder_custom_args_to_heterogeneous_map(); + decoder_ = decoder::get("nv-qldpc-decoder", H_tensor, params); + ASSERT_NE(decoder_, nullptr); + + decoder_->set_D_sparse(dec.D_sparse); + decoder_->set_O_sparse(dec.O_sparse); + + std::vector d_rp, d_ci; + realtime::sparse_vec_to_csr(dec.D_sparse, d_rp, d_ci); + num_measurements_ = 0; + for (auto c : d_ci) + num_measurements_ = std::max(num_measurements_, (std::size_t)(c + 1)); + + std::vector o_rp, o_ci; + num_observables_ = realtime::sparse_vec_to_csr(dec.O_sparse, o_rp, o_ci); + + printf("Config: block_size=%zu, syndrome_size=%zu, " + "num_measurements=%zu, num_observables=%zu\n", + bs, ss, num_measurements_, num_observables_); + + // --- Capture CUDA graph --- + ASSERT_TRUE(decoder_->supports_graph_dispatch()); + void *raw = decoder_->capture_decode_graph(); + ASSERT_NE(raw, nullptr); + graph_res_ = static_cast(raw); + ASSERT_NE(graph_res_->graph_exec, nullptr); + printf("Graph captured: function_id=0x%08X\n", graph_res_->function_id); + + // --- Load syndromes --- syndromes_ = load_syndromes(std::string(TEST_DATA_DIR) + "/syndromes_nv_qldpc_relay.txt", num_measurements_); printf("Loaded %zu test syndromes\n", syndromes_.size()); ASSERT_GT(syndromes_.size(), 0u); - // Per-round shape: take it from the first shot's slicing; assert - // remaining shots match (otherwise the fixture is internally - // inconsistent and the per-round dispatch contract has no meaning). - ASSERT_GT(syndromes_[0].per_round_measurements.size(), 0u); - num_rounds_ = syndromes_[0].per_round_measurements.size(); - measurements_per_round_ = syndromes_[0].per_round_measurements[0].size(); - ASSERT_GT(measurements_per_round_, 0u); - for (std::size_t i = 0; i < syndromes_.size(); ++i) { - ASSERT_EQ(syndromes_[i].per_round_measurements.size(), num_rounds_) - << "Shot " << i << " has " - << syndromes_[i].per_round_measurements.size() << " rounds; expected " - << num_rounds_; - for (std::size_t r = 0; r < num_rounds_; ++r) - ASSERT_EQ(syndromes_[i].per_round_measurements[r].size(), - measurements_per_round_) - << "Shot " << i << " round " << r << " has wrong measurement count"; - } - // Sanity-check that the per-round slicing covers exactly the same - // measurements the decoder's D matrix expects. - ASSERT_EQ(num_rounds_ * measurements_per_round_, num_measurements_); - printf("Per-shot shape: %zu rounds x %zu measurements/round = %zu total\n", - num_rounds_, measurements_per_round_, num_measurements_); - - // ---- Initialize the realtime session ---- - // - // The session needs the dispatch-kernel launch function pointer; that - // symbol lives in libcudaq-realtime-dispatch.a (a static archive with - // hidden visibility), which this exe -- and ONLY this exe -- links. - // The session's .so cannot reference it directly, so we hand it in as - // a constructor parameter. See qec_realtime_session.h for the - // rationale (cudaq_dispatch_launch_fn_t docstring). - session_ = std::make_unique( - decoders_, &cudaq_launch_dispatch_kernel_regular); - try { - session_->initialize(); - } catch (const std::exception &e) { - FAIL() << "qec_realtime_session::initialize threw: " << e.what(); - } - - // Under decoder_server_runtime.md every decoder shares the canonical - // enqueue_syndromes function_id; per-decoder routing is by routing_key. - // rpc_producer::enqueue_syndromes writes that canonical fid into the - // RPCHeader and `decoder_id_` into payload arg0 (which doubles as the - // routing_key the host monitor matches against the function table). - printf("Session initialized: enqueue_fn_id=0x%08X (canonical, " - "decoder_id=%zu), %zu slots\n", - cudaq::qec::decoding::rpc::kEnqueueSyndromesFunctionId, decoder_id_, - session_->num_slots()); + // --- Allocate ring buffers --- + slot_size_ = std::max(sizeof(RPCHeader) + num_measurements_, + sizeof(RPCResponse) + num_observables_); + slot_size_ = (slot_size_ + 255) & ~255u; + printf("Buffer size: %zu bytes\n", slot_size_); + + ASSERT_TRUE(allocate_ring_buffer(num_slots_, slot_size_, &rx_flags_host_, + &rx_flags_dev_, &rx_data_host_, + &rx_data_dev_)); + ASSERT_TRUE(allocate_ring_buffer(num_slots_, slot_size_, &tx_flags_host_, + &tx_flags_dev_, &tx_data_host_, + &tx_data_dev_)); + + // --- Wire the C API: HOST_LOOP dispatcher with separate RX/TX --- + memset(&ringbuffer_, 0, sizeof(ringbuffer_)); + ringbuffer_.rx_flags = rx_flags_dev_; + ringbuffer_.tx_flags = tx_flags_dev_; + ringbuffer_.rx_data = rx_data_dev_; + ringbuffer_.tx_data = tx_data_dev_; + ringbuffer_.rx_stride_sz = slot_size_; + ringbuffer_.tx_stride_sz = slot_size_; + ringbuffer_.rx_flags_host = rx_flags_host_; + ringbuffer_.tx_flags_host = tx_flags_host_; + ringbuffer_.rx_data_host = rx_data_host_; + ringbuffer_.tx_data_host = tx_data_host_; + + memset(host_table_, 0, sizeof(host_table_)); + host_table_[0].function_id = graph_res_->function_id; + host_table_[0].dispatch_mode = CUDAQ_DISPATCH_GRAPH_LAUNCH; + host_table_[0].handler.graph_exec = graph_res_->graph_exec; + + ASSERT_EQ(cudaq_dispatch_manager_create(&manager_), CUDAQ_OK); + + cudaq_dispatcher_config_t disp_config{}; + disp_config.device_id = 0; + disp_config.num_slots = static_cast(num_slots_); + disp_config.slot_size = static_cast(slot_size_); + disp_config.dispatch_path = CUDAQ_DISPATCH_PATH_HOST; + + ASSERT_EQ(cudaq_dispatcher_create(manager_, &disp_config, &dispatcher_), + CUDAQ_OK); + ASSERT_EQ(cudaq_dispatcher_set_ringbuffer(dispatcher_, &ringbuffer_), + CUDAQ_OK); + + cudaq_function_table_t table{}; + table.entries = host_table_; + table.count = 1; + ASSERT_EQ(cudaq_dispatcher_set_function_table(dispatcher_, &table), + CUDAQ_OK); + + shutdown_flag_ = 0; + stats_counter_ = 0; + ASSERT_EQ(cudaq_dispatcher_set_control(dispatcher_, &shutdown_flag_, + &stats_counter_), + CUDAQ_OK); + + ASSERT_EQ(cudaq_dispatcher_set_mailbox(dispatcher_, graph_res_->h_mailbox), + CUDAQ_OK); + + ASSERT_EQ(cudaq_dispatcher_start(dispatcher_), CUDAQ_OK); } void TearDown() override { - // Session finalize is idempotent + safe to call after a partially- - // initialized SetUp (some ASSERTs above can return before initialize() - // succeeds; the unique_ptr might still be null in that case). - if (session_) { - session_->finalize(); - session_.reset(); + if (dispatcher_) { + shutdown_flag_ = 1; + __sync_synchronize(); + cudaq_dispatcher_stop(dispatcher_); + cudaq_dispatcher_destroy(dispatcher_); + dispatcher_ = nullptr; + } + if (manager_) { + cudaq_dispatch_manager_destroy(manager_); + manager_ = nullptr; } + if (decoder_) + decoder_->release_decode_graph(graph_res_); + free_ring_buffer(rx_flags_host_, rx_data_host_); + free_ring_buffer(tx_flags_host_, tx_data_host_); + } - // Drop the decoder vector AFTER the session releases its captured - // graphs (session.finalize() above). Order matters because the - // session holds non-owning pointers into decoders_ for graph release. - decoders_.clear(); + static std::string read_file(const std::string &path) { + std::ifstream f(path); + return std::string((std::istreambuf_iterator(f)), + std::istreambuf_iterator()); } }; //============================================================================== // Test: Graph decode of all test syndromes via HOST_LOOP dispatch -// -// Post-Step-8 the test body is reduced to: build a payload, hand it to -// rpc_producer, validate the returned bytes against the fixture. All slot -// leasing / RPCHeader assembly / spin-on-magic logic now lives in -// rpc_producer.cpp and is shared with surface_code-1-local. //============================================================================== TEST_F(GraphDecodeTest, DecodesAllSyndromes) { - ASSERT_NE(session_, nullptr); - auto &session = *session_; - using namespace cudaq::qec::decoding; - - int enqueue_ok = 0; - int correction_matched = 0; - int correction_mismatched = 0; + int success_count = 0; + int error_count = 0; using clock_t = std::chrono::high_resolution_clock; std::vector shot_durations_us; shot_durations_us.reserve(syndromes_.size()); - // Re-usable correction output buffer (one shot's worth of observables). - std::vector corrections(num_observables_, 0); - for (std::size_t shot = 0; shot < syndromes_.size(); ++shot) { + uint32_t slot = static_cast(shot % num_slots_); + + // Wait for slot to be available (both rx and tx flags clear) + int timeout = 5000; + while (!cudaq_host_ringbuffer_slot_available(&ringbuffer_, slot) && + timeout-- > 0) + usleep(200); + ASSERT_GT(timeout, 0) << "Timeout waiting for slot " << slot << " at shot " + << shot; + + // Clear stale data in both RX and TX slots + memset(rx_data_host_ + slot * slot_size_, 0, slot_size_); + memset(tx_data_host_ + slot * slot_size_, 0, slot_size_); + + // Write RPC request into the RX ring buffer slot + ASSERT_EQ(cudaq_host_ringbuffer_write_rpc_request( + &ringbuffer_, slot, graph_res_->function_id, + syndromes_[shot].measurements.data(), + static_cast(syndromes_[shot].measurements.size()), + static_cast(shot), 0), + CUDAQ_OK); + auto t_start = clock_t::now(); - // ---------------------------------------------------------------------- - // (a) Per-round enqueue. Under decoder_server_runtime.md the - // dispatcher always emits an empty (result_len == 0) RPCResponse - // for enqueue_syndromes -- there is no did_decode flag on the - // wire anymore. Whether a window closed and a decode latched - // is observable post-suite through get_corrections(reset=1). - // ---------------------------------------------------------------------- - for (std::size_t round = 0; round < num_rounds_; ++round) { - const auto &round_bytes = syndromes_[shot].per_round_measurements[round]; - ASSERT_EQ(round_bytes.size(), measurements_per_round_); - - // `tag` carries (shot, round) for diagnostic / ordering checks. - // rpc_producer::enqueue_syndromes writes `tag` into BOTH: - // - payload arg1 (full 64 bits, per decoder_server_runtime.md - // #enqueue_syndromes), and - // - RPCHeader::request_id (low 32 bits, for the realtime layer's - // id-echo correlation). - // Either path is sufficient to correlate the response with the - // request. - const std::uint64_t tag = (static_cast(shot) << 16) | - static_cast(round); - - try { - rpc_producer::enqueue_syndromes( - session, decoder_id_, round_bytes.data(), - static_cast(measurements_per_round_), tag); - } catch (const std::exception &e) { - FAIL() << "rpc_producer::enqueue_syndromes threw at shot " << shot - << " round " << round << ": " << e.what(); - } - enqueue_ok++; - } + // Signal the slot (host dispatcher picks it up) + cudaq_host_ringbuffer_signal_slot(&ringbuffer_, slot); - // ---------------------------------------------------------------------- - // (b) get_corrections. reset=1 zeros corrections after fetch so the - // next shot starts from cleared state on the device side. The - // plugin's BP iteration warm-start is governed by reset_decoder - // (issued at end-of-suite below), not by this reset flag. - // ---------------------------------------------------------------------- - std::fill(corrections.begin(), corrections.end(), - static_cast(0xCC)); // poison - try { - rpc_producer::get_corrections( - session, decoder_id_, corrections.data(), - static_cast(num_observables_), - /*reset=*/1); - } catch (const std::exception &e) { - FAIL() << "rpc_producer::get_corrections threw at shot " << shot << ": " - << e.what(); + // Poll for READY -- the graph kernel signals via tx_flag + int cuda_err = 0; + cudaq_tx_status_t st = CUDAQ_TX_EMPTY; + for (int i = 0; i < 50000 && st != CUDAQ_TX_READY; ++i) { + usleep(200); + st = cudaq_host_ringbuffer_poll_tx_flag(&ringbuffer_, slot, &cuda_err); } - const std::uint8_t got = corrections[0]; - const std::uint8_t expected = syndromes_[shot].expected_correction; - const bool matches = (got == expected); - if (matches) - correction_matched++; - else - correction_mismatched++; - EXPECT_EQ(got, expected) - << "Correction byte 0 mismatch at shot " << shot - << " (got=" << static_cast(got) - << ", expected=" << static_cast(expected) << ")"; + ASSERT_EQ(st, CUDAQ_TX_READY) + << "Expected READY from graph kernel at shot " << shot << " (got " << st + << ", cuda_err=" << cuda_err << ")"; + + CUDA_CHECK(cudaDeviceSynchronize()); auto t_end = clock_t::now(); double duration_us = std::chrono::duration(t_end - t_start).count(); shot_durations_us.push_back(duration_us); - printf("Shot %zu: enqueue_rounds=%zu, corr=[", shot, num_rounds_); - for (std::size_t i = 0; i < num_observables_ && i < 8; i++) { + // Read response from the TX buffer (separate from RX) + __sync_synchronize(); + uint8_t *slot_data = tx_data_host_ + slot * slot_size_; + auto *response = reinterpret_cast(slot_data); + + ASSERT_EQ(response->magic, RPC_MAGIC_RESPONSE) + << "Bad response magic for shot " << shot; + EXPECT_EQ(response->status, 0) << "Non-zero status for shot " << shot; + EXPECT_EQ(response->result_len, static_cast(num_observables_)) + << "Wrong result_len for shot " << shot; + EXPECT_EQ(response->request_id, static_cast(shot)) + << "request_id mismatch for shot " << shot; + + const uint8_t *corrections = slot_data + sizeof(RPCResponse); + + if (response->status == 0 && + response->result_len == static_cast(num_observables_)) { + success_count++; + } else { + error_count++; + } + + printf("Shot %zu: status=%d, result_len=%u, time=%.1f us, corrections=[", + shot, response->status, response->result_len, duration_us); + for (uint32_t i = 0; i < response->result_len && i < 8; i++) { printf("%u", corrections[i]); - if (i + 1 < num_observables_ && i + 1 < 8) + if (i + 1 < response->result_len) printf(","); } - printf("] expected=%u %s (%.1f us shot-total)\n", - static_cast(expected), matches ? "OK" : "MISMATCH", - duration_us); - } - - // -------------------------------------------------------------------------- - // (c) End-of-suite reset_decoder + get_corrections to verify the reset - // path actually zeros device-side state. Without this, a regression - // where reset is a no-op would still pass the per-shot checks - // (because we already pass reset=1 to get_corrections; that uses a - // different code path inside the plugin). - // -------------------------------------------------------------------------- - try { - rpc_producer::reset_decoder(*session_, decoder_id_); - } catch (const std::exception &e) { - FAIL() << "rpc_producer::reset_decoder threw at end-of-suite: " << e.what(); - } + printf("]\n"); - std::fill(corrections.begin(), corrections.end(), - static_cast(0xCC)); - try { - rpc_producer::get_corrections(*session_, decoder_id_, corrections.data(), - static_cast(num_observables_), - /*reset=*/0); - } catch (const std::exception &e) { - FAIL() << "rpc_producer::get_corrections post-reset threw: " << e.what(); + // Release the worker and clear the slot for reuse + cudaq_host_release_worker(dispatcher_, 0); + cudaq_host_ringbuffer_clear_slot(&ringbuffer_, slot); } - for (std::size_t i = 0; i < num_observables_; ++i) - EXPECT_EQ(corrections[i], 0) - << "reset_decoder did not zero corrections[" << i << "]"; - printf("\nCompleted: enqueue OK = %d (expected %zu)\n", enqueue_ok, - syndromes_.size() * num_rounds_); - printf("Correction comparison vs fixture: %d matched, %d mismatched " - "(over %zu shots)\n", - correction_matched, correction_mismatched, syndromes_.size()); + printf("\nCompleted: %d/%zu shots successful, %d errors\n", success_count, + syndromes_.size(), error_count); + // Timing summary (skip shot 0 as warmup) if (shot_durations_us.size() > 1) { auto begin = shot_durations_us.begin() + 1; auto end = shot_durations_us.end(); @@ -484,7 +446,6 @@ TEST_F(GraphDecodeTest, DecodesAllSyndromes) { min_val, avg, max_val); } - EXPECT_EQ(enqueue_ok, static_cast(syndromes_.size() * num_rounds_)); - EXPECT_EQ(correction_matched, static_cast(syndromes_.size())); - EXPECT_EQ(correction_mismatched, 0); + EXPECT_EQ(success_count, static_cast(syndromes_.size())); + EXPECT_EQ(error_count, 0); } diff --git a/libs/qec/unittests/realtime/qec_roce_decode_test/data/config_nv_qldpc_relay.yml b/libs/qec/unittests/realtime/qec_roce_decode_test/data/config_nv_qldpc_relay.yml index c9c489a4a..2b71c8470 100644 --- a/libs/qec/unittests/realtime/qec_roce_decode_test/data/config_nv_qldpc_relay.yml +++ b/libs/qec/unittests/realtime/qec_roce_decode_test/data/config_nv_qldpc_relay.yml @@ -28,6 +28,4 @@ decoders: stopping_criterion: All stop_nconv: 1 composition: 1 - clip_value: 200.0 - repeatable: true ... diff --git a/libs/qec/unittests/realtime/qec_roce_decode_test/data/syndromes_nv_qldpc_relay.txt b/libs/qec/unittests/realtime/qec_roce_decode_test/data/syndromes_nv_qldpc_relay.txt index d431d2b78..bc2b08d35 100644 --- a/libs/qec/unittests/realtime/qec_roce_decode_test/data/syndromes_nv_qldpc_relay.txt +++ b/libs/qec/unittests/realtime/qec_roce_decode_test/data/syndromes_nv_qldpc_relay.txt @@ -1,54 +1,39 @@ NUM_DATA 9 NUM_LOGICAL 1 SHOT_START 0 -ROUND_START 0 0 0 0 0 -1 -1 -1 0 -ROUND_START 1 0 0 +1 0 0 -1 -1 -1 0 -ROUND_START 2 0 0 0 0 1 -1 -1 0 -ROUND_START 3 0 0 0 0 -1 0 0 +1 0 -SHOT_START 1 -ROUND_START 4 0 0 0 0 0 -1 0 1 -ROUND_START 5 -0 +SHOT_START 1 0 0 0 @@ -56,7 +41,6 @@ ROUND_START 5 1 0 1 -ROUND_START 6 0 0 0 @@ -65,7 +49,6 @@ ROUND_START 6 1 0 1 -ROUND_START 7 0 0 0 @@ -74,304 +57,265 @@ ROUND_START 7 1 0 1 -SHOT_START 2 -ROUND_START 8 -0 0 0 0 0 0 +1 0 +1 0 -ROUND_START 9 +SHOT_START 2 0 0 0 0 +1 +1 0 0 0 0 -ROUND_START 10 0 0 +1 +1 0 0 0 0 0 0 -ROUND_START 11 +1 +1 0 0 0 0 0 0 +1 +1 0 0 SHOT_START 3 -ROUND_START 12 0 0 0 0 -1 0 1 0 -ROUND_START 13 0 0 0 0 -1 +0 0 1 0 -ROUND_START 14 0 0 0 0 -1 +0 0 1 0 -ROUND_START 15 0 0 0 0 -1 +0 0 1 0 -SHOT_START 4 -ROUND_START 16 0 +SHOT_START 4 0 0 0 -1 0 1 1 -ROUND_START 17 0 0 0 0 -1 0 +0 +1 1 0 -ROUND_START 18 0 0 0 0 -1 0 1 +1 0 -ROUND_START 19 0 0 0 0 -1 0 1 +1 0 -SHOT_START 5 -ROUND_START 20 0 +SHOT_START 5 0 0 0 0 1 +1 0 0 -ROUND_START 21 -0 0 0 0 0 1 +1 0 0 -ROUND_START 22 -0 0 0 0 0 1 +1 0 0 -ROUND_START 23 -0 0 0 0 0 1 +1 0 0 SHOT_START 6 -ROUND_START 24 0 0 0 0 0 1 +0 +1 1 0 -ROUND_START 25 0 0 +1 +1 +1 +1 +1 0 0 0 1 -1 -0 -ROUND_START 26 0 +1 0 +1 0 0 0 1 +0 1 0 -ROUND_START 27 +SHOT_START 7 +0 0 -1 0 0 0 -1 1 0 -SHOT_START 7 -ROUND_START 28 +0 0 0 0 0 0 1 -1 -1 -ROUND_START 29 0 0 0 0 0 -1 -1 -1 -ROUND_START 30 0 0 1 0 0 -1 -1 0 -ROUND_START 31 0 0 -1 0 0 1 -1 +0 0 SHOT_START 8 -ROUND_START 32 0 0 0 0 -1 0 -1 0 -ROUND_START 33 0 0 0 0 -1 0 -1 0 -ROUND_START 34 0 0 0 0 -1 0 -1 0 -ROUND_START 35 0 0 0 0 -1 0 -1 0 -SHOT_START 9 -ROUND_START 36 0 0 0 0 0 -1 0 -1 -ROUND_START 37 +0 +0 +SHOT_START 9 0 0 0 0 0 -1 0 1 -ROUND_START 38 0 0 0 0 0 -1 -1 -1 -ROUND_START 39 0 0 +1 0 0 0 -1 -1 -1 -SHOT_START 10 -ROUND_START 40 0 0 0 @@ -380,341 +324,317 @@ ROUND_START 40 0 0 0 -ROUND_START 41 0 0 0 0 1 0 -0 -0 -ROUND_START 42 +SHOT_START 10 0 0 0 0 1 +1 +1 +1 0 0 0 -ROUND_START 43 +0 +1 +1 +1 +1 0 0 0 0 1 +1 +1 +1 +0 0 0 0 +1 +1 +1 +1 SHOT_START 11 -ROUND_START 44 0 0 0 0 0 1 +1 0 0 -ROUND_START 45 -0 0 0 0 0 1 +1 0 0 -ROUND_START 46 -0 0 0 0 0 1 +1 0 0 -ROUND_START 47 -0 0 0 0 0 1 -0 +1 0 SHOT_START 12 -ROUND_START 48 0 0 0 0 -0 -1 1 1 -ROUND_START 49 -1 +0 +0 0 0 0 0 1 1 -1 -ROUND_START 50 -1 0 0 +1 +0 0 0 1 1 -1 -ROUND_START 51 -1 0 0 +1 +0 0 0 1 1 -1 -SHOT_START 13 -ROUND_START 52 0 0 +SHOT_START 13 0 0 0 -1 0 0 -ROUND_START 53 0 0 0 0 0 -1 0 0 -ROUND_START 54 0 0 0 0 0 -1 0 0 -ROUND_START 55 0 0 0 0 0 -1 0 0 -SHOT_START 14 -ROUND_START 56 0 0 0 0 0 -1 -1 -1 -ROUND_START 57 0 +SHOT_START 14 0 0 0 0 1 +0 +0 1 -1 -ROUND_START 58 0 0 0 0 +1 +1 +0 +1 +0 0 1 1 1 -ROUND_START 59 1 0 +1 0 0 -0 1 1 1 +1 +0 +1 SHOT_START 15 -ROUND_START 60 0 0 0 0 1 1 -1 -1 -ROUND_START 61 +0 +0 0 0 0 0 1 1 -1 -1 -ROUND_START 62 0 -1 0 0 -1 +0 +0 0 1 1 -ROUND_START 63 0 -1 0 0 -1 +0 +0 0 1 1 +0 +0 SHOT_START 16 -ROUND_START 64 0 0 0 0 0 0 -1 0 -ROUND_START 65 0 0 0 0 0 0 -1 0 -ROUND_START 66 0 -1 0 0 0 0 -1 0 -ROUND_START 67 0 -1 0 0 0 0 -1 0 -SHOT_START 17 -ROUND_START 68 0 0 0 0 -1 -1 0 0 -ROUND_START 69 +SHOT_START 17 +0 +0 +0 0 0 0 0 -1 1 0 0 -ROUND_START 70 0 0 0 0 -1 +0 1 0 0 -ROUND_START 71 +0 0 0 0 0 1 -1 0 0 +0 +0 +0 +0 +0 +1 SHOT_START 18 -ROUND_START 72 0 0 0 0 1 +0 1 1 0 -ROUND_START 73 -0 0 0 0 1 +0 1 1 0 -ROUND_START 74 0 0 0 1 -1 +0 1 1 0 -ROUND_START 75 0 0 0 1 +0 1 1 -1 -0 SHOT_START 19 -ROUND_START 76 -0 -0 0 0 0 0 0 1 -ROUND_START 77 -0 +1 0 0 0 @@ -722,8 +642,7 @@ ROUND_START 77 0 0 1 -ROUND_START 78 -0 +1 0 0 0 @@ -731,8 +650,7 @@ ROUND_START 78 0 0 1 -ROUND_START 79 -0 +1 0 0 0 @@ -740,8 +658,10 @@ ROUND_START 79 0 0 1 +1 +0 SHOT_START 20 -ROUND_START 80 +0 0 0 0 @@ -749,36 +669,13 @@ ROUND_START 80 1 0 0 -1 -ROUND_START 81 -1 0 -1 0 -1 -1 -1 -1 -ROUND_START 82 -1 0 -1 0 -1 -1 -1 -1 -ROUND_START 83 -1 0 1 0 -1 -1 -1 -1 -SHOT_START 21 -ROUND_START 84 0 0 0 @@ -786,8 +683,7 @@ ROUND_START 84 0 0 1 -1 -ROUND_START 85 +0 0 0 0 @@ -795,64 +691,63 @@ ROUND_START 85 0 0 1 -1 -ROUND_START 86 0 0 +SHOT_START 21 0 0 0 0 1 1 -ROUND_START 87 -0 -0 +1 +1 0 0 0 0 1 1 -SHOT_START 22 -ROUND_START 88 -0 +1 +1 0 0 0 0 1 1 +1 +1 0 -ROUND_START 89 0 0 0 +1 +1 +1 +1 +SHOT_START 22 +0 +0 0 0 -1 1 0 -ROUND_START 90 0 0 0 0 0 -1 +0 1 0 -ROUND_START 91 0 0 0 0 0 -1 -1 0 -SHOT_START 23 -ROUND_START 92 +1 0 0 0 @@ -861,35 +756,43 @@ ROUND_START 92 0 0 1 -ROUND_START 93 0 0 0 +SHOT_START 23 0 0 0 0 1 -ROUND_START 94 -0 0 +1 +1 0 0 0 0 +1 0 1 -ROUND_START 95 +1 0 0 0 0 +1 0 +1 +1 +0 +1 0 0 1 +1 +1 +1 SHOT_START 24 -ROUND_START 96 0 0 0 @@ -898,7 +801,6 @@ ROUND_START 96 1 1 0 -ROUND_START 97 0 0 0 @@ -907,18 +809,16 @@ ROUND_START 97 1 1 0 -ROUND_START 98 0 -1 +0 0 0 0 1 1 0 -ROUND_START 99 0 -1 +0 0 0 0 @@ -926,118 +826,95 @@ ROUND_START 99 1 0 SHOT_START 25 -ROUND_START 100 0 0 0 0 -1 +0 0 0 1 -ROUND_START 101 0 0 0 0 -1 +0 0 0 1 -ROUND_START 102 0 0 0 0 -1 +0 0 0 1 -ROUND_START 103 0 0 0 0 -1 +0 0 0 1 SHOT_START 26 -ROUND_START 104 0 0 0 0 -1 0 1 +0 1 -ROUND_START 105 0 0 0 0 -1 +0 0 1 1 -ROUND_START 106 0 0 0 0 -1 +0 0 1 1 -ROUND_START 107 0 0 0 0 -1 +0 0 1 1 SHOT_START 27 -ROUND_START 108 0 0 0 0 1 1 -0 1 -ROUND_START 109 0 0 0 0 -1 -1 0 1 -ROUND_START 110 -0 -0 -0 -0 1 1 0 -1 -ROUND_START 111 0 0 0 0 1 1 -0 1 -SHOT_START 28 -ROUND_START 112 0 0 0 @@ -1046,35 +923,22 @@ ROUND_START 112 1 1 1 -ROUND_START 113 0 +SHOT_START 28 0 0 0 0 -1 -1 -1 -ROUND_START 114 0 0 0 0 0 -1 -1 -1 -ROUND_START 115 0 0 0 0 0 -1 -1 -1 -SHOT_START 29 -ROUND_START 116 0 0 0 @@ -1083,7 +947,6 @@ ROUND_START 116 0 0 0 -ROUND_START 117 0 0 0 @@ -1092,45 +955,45 @@ ROUND_START 117 0 0 0 -ROUND_START 118 0 0 +SHOT_START 29 0 0 0 0 0 0 -ROUND_START 119 +1 +1 0 0 0 0 0 0 +1 +1 0 0 -SHOT_START 30 -ROUND_START 120 0 0 0 0 1 +1 0 0 0 -ROUND_START 121 -0 0 0 0 1 +1 +SHOT_START 30 0 0 0 -ROUND_START 122 -0 0 0 0 @@ -1138,7 +1001,6 @@ ROUND_START 122 0 0 0 -ROUND_START 123 0 0 0 @@ -1147,45 +1009,45 @@ ROUND_START 123 0 0 0 -SHOT_START 31 -ROUND_START 124 0 0 0 0 1 +0 +0 +0 +1 1 +0 1 0 -ROUND_START 125 +0 +SHOT_START 31 0 0 0 0 1 -1 +0 1 0 -ROUND_START 126 0 0 0 0 1 -1 +0 1 0 -ROUND_START 127 0 0 0 0 1 -1 +0 1 0 -SHOT_START 32 -ROUND_START 128 0 0 0 @@ -1194,35 +1056,40 @@ ROUND_START 128 0 1 0 -ROUND_START 129 +SHOT_START 32 0 0 0 0 1 -0 +1 1 0 -ROUND_START 130 -0 +1 0 +1 0 +1 +1 +1 0 1 0 1 0 -ROUND_START 131 +1 1 1 0 +1 0 1 0 1 +1 +1 0 SHOT_START 33 -ROUND_START 132 0 0 0 @@ -1231,7 +1098,6 @@ ROUND_START 132 1 1 1 -ROUND_START 133 0 0 0 @@ -1240,7 +1106,6 @@ ROUND_START 133 1 1 1 -ROUND_START 134 0 0 0 @@ -1249,7 +1114,6 @@ ROUND_START 134 1 1 1 -ROUND_START 135 0 0 0 @@ -1259,182 +1123,177 @@ ROUND_START 135 1 1 SHOT_START 34 -ROUND_START 136 0 0 0 0 0 1 -0 1 -ROUND_START 137 +1 0 0 0 0 0 1 -0 1 -ROUND_START 138 +1 0 0 0 0 0 1 -0 1 -ROUND_START 139 +1 0 0 0 0 0 1 -0 +1 1 SHOT_START 35 -ROUND_START 140 -0 -0 0 0 0 0 1 1 -ROUND_START 141 -0 -0 +1 +1 0 0 0 0 1 1 -ROUND_START 142 -0 -0 +1 +1 0 0 0 0 1 1 -ROUND_START 143 -0 -0 +1 +1 0 0 0 0 1 1 +1 +1 SHOT_START 36 -ROUND_START 144 0 0 0 0 -1 0 1 +1 +1 0 -ROUND_START 145 0 0 0 0 1 -0 +1 1 0 -ROUND_START 146 0 0 -1 +0 0 1 1 1 -1 -ROUND_START 147 0 0 -1 0 -1 +0 +0 1 1 1 SHOT_START 37 -ROUND_START 148 0 0 0 0 +1 0 +1 0 0 0 -ROUND_START 149 0 0 +1 0 +1 0 0 0 0 0 -ROUND_START 150 +1 0 +1 0 0 0 0 0 +1 0 +1 0 -ROUND_START 151 +SHOT_START 38 0 0 0 0 +1 0 0 +1 0 0 -SHOT_START 38 -ROUND_START 152 0 0 +1 +0 +0 +1 0 0 0 0 1 -1 -ROUND_START 153 0 0 +1 0 0 0 0 1 -1 -ROUND_START 154 0 0 +1 +SHOT_START 39 0 0 0 0 1 -1 -ROUND_START 155 +0 0 0 0 @@ -1443,165 +1302,130 @@ ROUND_START 155 0 1 1 -SHOT_START 39 -ROUND_START 156 0 0 0 0 -1 +0 0 1 1 -ROUND_START 157 -0 0 0 +1 0 1 0 1 +0 1 -ROUND_START 158 0 +SHOT_START 40 0 0 0 -1 0 -1 -1 -ROUND_START 159 0 0 0 0 -1 0 -1 -1 -SHOT_START 40 -ROUND_START 160 0 0 0 0 0 -1 -1 -1 -ROUND_START 161 0 0 0 0 0 -1 -1 -1 -ROUND_START 162 0 0 0 0 0 -1 -1 -1 -ROUND_START 163 0 0 0 0 0 1 -1 +0 1 SHOT_START 41 -ROUND_START 164 0 0 0 0 1 1 -0 1 -ROUND_START 165 +1 0 0 0 0 1 1 -0 1 -ROUND_START 166 +1 0 0 0 0 1 1 -0 1 -ROUND_START 167 -0 +1 0 +1 0 0 1 1 -0 +1 1 SHOT_START 42 -ROUND_START 168 0 0 0 0 1 0 -0 1 -ROUND_START 169 +1 0 0 0 0 1 0 -0 1 -ROUND_START 170 +1 0 0 0 0 1 0 -0 1 -ROUND_START 171 +1 0 0 0 0 1 -0 -0 +1 +1 1 SHOT_START 43 -ROUND_START 172 -0 -0 0 0 0 0 0 1 -ROUND_START 173 0 0 0 @@ -1610,7 +1434,6 @@ ROUND_START 173 0 0 1 -ROUND_START 174 0 0 0 @@ -1619,7 +1442,6 @@ ROUND_START 174 0 0 1 -ROUND_START 175 0 0 0 @@ -1628,8 +1450,9 @@ ROUND_START 175 0 0 1 +0 +0 SHOT_START 44 -ROUND_START 176 0 0 0 @@ -1637,8 +1460,7 @@ ROUND_START 176 1 1 1 -0 -ROUND_START 177 +1 0 0 0 @@ -1646,8 +1468,7 @@ ROUND_START 177 1 1 1 -0 -ROUND_START 178 +1 0 0 0 @@ -1655,8 +1476,7 @@ ROUND_START 178 1 1 1 -0 -ROUND_START 179 +1 0 0 0 @@ -1664,46 +1484,50 @@ ROUND_START 179 1 1 1 -0 +1 SHOT_START 45 -ROUND_START 180 0 0 0 0 0 0 +1 +1 0 0 -ROUND_START 181 0 0 0 0 +1 +1 0 0 0 0 -ROUND_START 182 0 0 +1 +1 0 0 0 0 0 0 -ROUND_START 183 +1 +1 +SHOT_START 46 0 0 0 0 0 +1 0 0 0 -SHOT_START 46 -ROUND_START 184 0 0 0 @@ -1711,26 +1535,15 @@ ROUND_START 184 1 0 0 -1 -ROUND_START 185 -0 0 0 0 -1 0 0 1 -ROUND_START 186 -0 -0 0 0 -1 -0 0 -1 -ROUND_START 187 0 0 0 @@ -1738,83 +1551,73 @@ ROUND_START 187 1 0 0 -1 SHOT_START 47 -ROUND_START 188 0 0 0 0 0 +1 0 +1 0 0 -ROUND_START 189 0 0 0 +1 0 +1 0 0 0 0 -ROUND_START 190 0 +1 0 +1 0 0 0 0 0 +1 0 -ROUND_START 191 +1 +SHOT_START 48 0 0 0 0 0 +1 0 0 0 -SHOT_START 48 -ROUND_START 192 0 0 0 0 1 -1 0 0 -ROUND_START 193 0 0 0 0 -1 -1 0 0 -ROUND_START 194 0 -1 0 0 -1 -1 0 0 -ROUND_START 195 0 -1 0 0 -1 -1 0 0 SHOT_START 49 -ROUND_START 196 0 0 0 @@ -1822,45 +1625,38 @@ ROUND_START 196 0 1 0 -1 -ROUND_START 197 -0 0 +1 0 0 0 -1 0 1 -ROUND_START 198 0 0 +1 0 0 0 -1 0 1 -ROUND_START 199 0 0 +1 0 0 0 -1 0 1 -SHOT_START 50 -ROUND_START 200 0 0 +SHOT_START 50 0 0 0 0 1 1 -ROUND_START 201 0 0 0 @@ -1869,7 +1665,6 @@ ROUND_START 201 0 1 1 -ROUND_START 202 0 0 0 @@ -1878,35 +1673,18 @@ ROUND_START 202 0 1 1 -ROUND_START 203 -0 -0 0 0 0 0 -1 -1 -SHOT_START 51 -ROUND_START 204 -0 0 0 -0 -1 1 1 0 -ROUND_START 205 -0 -0 0 +SHOT_START 51 0 -1 -1 -1 -0 -ROUND_START 206 0 0 0 @@ -1915,7 +1693,6 @@ ROUND_START 206 1 1 0 -ROUND_START 207 0 0 0 @@ -1924,27 +1701,13 @@ ROUND_START 207 1 1 0 -SHOT_START 52 -ROUND_START 208 -0 -0 0 0 0 0 1 1 -ROUND_START 209 -0 -0 -0 -0 -0 -0 1 -1 -ROUND_START 210 -0 0 0 0 @@ -1952,8 +1715,8 @@ ROUND_START 210 0 1 1 -ROUND_START 211 -0 +1 +SHOT_START 52 0 0 0 @@ -1961,83 +1724,72 @@ ROUND_START 211 0 1 1 -SHOT_START 53 -ROUND_START 212 0 0 0 0 -1 0 0 +1 +1 0 -ROUND_START 213 0 0 0 0 -1 0 +1 +1 0 0 -ROUND_START 214 0 0 0 0 1 +1 0 +SHOT_START 53 0 0 -ROUND_START 215 -0 0 0 0 1 +1 +1 0 0 0 -SHOT_START 54 -ROUND_START 216 -0 -0 0 0 1 1 -0 1 -ROUND_START 217 +0 0 0 0 0 1 1 -0 1 -ROUND_START 218 +0 0 0 0 0 1 1 -0 1 -ROUND_START 219 +SHOT_START 54 0 0 0 0 -1 -1 0 1 -SHOT_START 55 -ROUND_START 220 -0 +1 0 0 0 @@ -2045,8 +1797,7 @@ ROUND_START 220 0 0 1 -ROUND_START 221 -0 +1 0 0 0 @@ -2054,8 +1805,7 @@ ROUND_START 221 0 0 1 -ROUND_START 222 -0 +1 0 0 0 @@ -2063,91 +1813,108 @@ ROUND_START 222 0 0 1 -ROUND_START 223 -0 +1 0 +SHOT_START 55 0 0 0 0 +1 0 1 -SHOT_START 56 -ROUND_START 224 +1 0 0 0 0 1 0 +1 +1 0 0 -ROUND_START 225 +1 0 +1 0 +1 +1 0 0 1 0 +1 0 -0 -ROUND_START 226 +1 +1 +SHOT_START 56 0 0 0 0 1 +1 0 +1 +1 0 +1 0 -ROUND_START 227 +1 +1 0 +1 +1 0 +1 0 +1 +1 0 1 +1 0 +1 0 +1 +1 0 +1 SHOT_START 57 -ROUND_START 228 0 0 0 0 1 -0 +1 0 1 -ROUND_START 229 0 0 0 0 1 -0 +1 0 1 -ROUND_START 230 0 0 0 0 1 -0 +1 0 1 -ROUND_START 231 0 0 0 0 1 -0 +1 0 1 SHOT_START 58 -ROUND_START 232 0 0 0 @@ -2156,7 +1923,6 @@ ROUND_START 232 0 1 0 -ROUND_START 233 0 0 0 @@ -2165,7 +1931,6 @@ ROUND_START 233 0 1 0 -ROUND_START 234 0 0 0 @@ -2174,7 +1939,6 @@ ROUND_START 234 0 1 0 -ROUND_START 235 0 0 0 @@ -2184,7 +1948,7 @@ ROUND_START 235 1 0 SHOT_START 59 -ROUND_START 236 +0 0 0 0 @@ -2193,7 +1957,6 @@ ROUND_START 236 1 1 0 -ROUND_START 237 0 0 0 @@ -2202,7 +1965,6 @@ ROUND_START 237 1 1 0 -ROUND_START 238 0 0 0 @@ -2211,7 +1973,6 @@ ROUND_START 238 1 1 0 -ROUND_START 239 0 0 0 @@ -2219,268 +1980,225 @@ ROUND_START 239 0 1 1 -0 SHOT_START 60 -ROUND_START 240 0 0 0 0 1 -0 1 0 -ROUND_START 241 +1 0 0 0 0 1 -0 1 0 -ROUND_START 242 +1 0 0 0 0 1 -0 1 0 -ROUND_START 243 +1 0 0 0 0 1 -0 1 0 +1 SHOT_START 61 -ROUND_START 244 0 0 0 0 -1 -1 -1 0 -ROUND_START 245 0 0 +1 0 0 -1 -1 -1 0 -ROUND_START 246 0 0 0 0 1 -1 -1 0 -ROUND_START 247 0 0 0 0 -1 -1 +0 +0 1 0 -SHOT_START 62 -ROUND_START 248 0 0 0 0 0 -1 0 1 -ROUND_START 249 +SHOT_START 62 +0 0 0 0 0 0 -1 0 -1 -ROUND_START 250 0 0 0 0 0 -1 0 -1 -ROUND_START 251 0 0 0 0 0 -1 0 -1 -SHOT_START 63 -ROUND_START 252 0 0 0 0 0 -1 -1 0 -ROUND_START 253 0 0 0 0 0 -1 -1 0 -ROUND_START 254 0 +SHOT_START 63 0 0 0 0 1 -1 0 -ROUND_START 255 +1 +1 0 0 0 0 +1 0 1 1 0 -SHOT_START 64 -ROUND_START 256 0 0 0 +1 0 1 +1 0 0 0 -ROUND_START 257 -1 0 +1 0 +1 +1 +SHOT_START 64 0 0 0 0 0 -ROUND_START 258 1 0 +1 0 0 0 0 0 +1 0 -ROUND_START 259 1 0 0 0 0 0 +1 0 +1 0 -SHOT_START 65 -ROUND_START 260 0 0 0 0 1 0 -0 -0 -ROUND_START 261 +1 +SHOT_START 65 0 0 0 0 1 0 +1 0 0 -ROUND_START 262 -0 0 0 0 1 0 +1 0 +1 0 -ROUND_START 263 0 0 0 0 1 0 +1 0 0 -SHOT_START 66 -ROUND_START 264 0 0 0 +1 0 +SHOT_START 66 0 -1 -1 0 -ROUND_START 265 0 0 0 0 0 -1 -1 0 -ROUND_START 266 0 0 0 0 0 -1 -1 0 -ROUND_START 267 0 0 0 0 0 -1 -1 0 -SHOT_START 67 -ROUND_START 268 0 0 0 @@ -2489,44 +2207,53 @@ ROUND_START 268 0 0 0 -ROUND_START 269 0 0 0 0 +SHOT_START 67 0 0 0 0 -ROUND_START 270 0 +1 +1 +1 0 0 0 0 0 +1 +1 +1 0 0 -ROUND_START 271 0 0 0 +1 +1 +1 0 0 0 0 0 +1 +1 +1 SHOT_START 68 -ROUND_START 272 0 0 0 0 +1 0 0 1 0 -ROUND_START 273 0 0 0 @@ -2535,7 +2262,6 @@ ROUND_START 273 0 1 0 -ROUND_START 274 0 0 0 @@ -2544,7 +2270,6 @@ ROUND_START 274 0 1 0 -ROUND_START 275 0 0 0 @@ -2552,416 +2277,359 @@ ROUND_START 275 0 0 1 -0 SHOT_START 69 -ROUND_START 276 -0 0 0 0 0 1 -0 1 -ROUND_START 277 +1 0 0 0 0 0 1 -0 1 -ROUND_START 278 +1 0 0 0 0 0 1 -0 1 -ROUND_START 279 +1 0 0 0 0 0 1 -0 1 +1 +0 SHOT_START 70 -ROUND_START 280 0 0 0 0 1 -1 +0 0 1 -ROUND_START 281 0 0 0 0 1 -1 +0 0 1 -ROUND_START 282 0 0 0 0 1 -1 +0 0 1 -ROUND_START 283 0 0 0 0 1 -1 +0 0 1 SHOT_START 71 -ROUND_START 284 0 0 0 0 0 1 -0 1 -ROUND_START 285 +1 0 0 0 0 0 1 -0 1 -ROUND_START 286 +1 0 0 0 0 0 1 -0 1 -ROUND_START 287 +1 0 0 0 0 0 1 -0 +1 1 SHOT_START 72 -ROUND_START 288 0 0 0 0 -1 0 0 1 -ROUND_START 289 0 0 0 0 -1 +0 0 0 1 -ROUND_START 290 0 0 0 0 -1 +0 0 0 1 -ROUND_START 291 0 0 0 0 -1 +0 0 0 1 +0 SHOT_START 73 -ROUND_START 292 0 0 0 0 1 +1 0 0 -1 -ROUND_START 293 0 0 0 0 1 +1 0 0 -1 -ROUND_START 294 0 0 0 0 1 +1 0 0 -1 -ROUND_START 295 0 0 0 0 1 +1 0 0 -1 SHOT_START 74 -ROUND_START 296 0 0 0 0 1 -1 -1 -1 -ROUND_START 297 -0 0 0 0 -1 -1 -1 -1 -ROUND_START 298 0 0 0 0 1 -1 -1 -1 -ROUND_START 299 0 0 0 0 -1 -1 -1 -1 -SHOT_START 75 -ROUND_START 300 0 0 0 +1 0 0 0 0 0 -ROUND_START 301 0 0 +1 0 0 0 +SHOT_START 75 0 0 0 -ROUND_START 302 0 0 0 +1 0 0 0 0 0 -ROUND_START 303 0 0 +1 0 0 0 +1 0 0 0 -SHOT_START 76 -ROUND_START 304 +1 +1 0 0 +1 0 0 0 1 1 -1 -ROUND_START 305 +SHOT_START 76 0 0 0 0 0 1 -1 -1 -ROUND_START 306 +0 +0 0 0 0 0 0 1 -1 -1 -ROUND_START 307 +0 +0 0 0 0 0 0 1 +0 +0 +0 +0 +0 +0 +0 1 -1 +0 +0 SHOT_START 77 -ROUND_START 308 0 0 0 0 0 +1 0 +1 0 0 -ROUND_START 309 0 0 0 +1 0 +1 0 0 0 0 -ROUND_START 310 0 +1 0 +1 0 0 0 0 0 +1 0 -ROUND_START 311 +1 +SHOT_START 78 0 0 0 0 0 0 +1 0 0 -SHOT_START 78 -ROUND_START 312 0 0 0 0 -1 -1 0 1 -ROUND_START 313 0 0 0 0 -1 -1 0 -1 -ROUND_START 314 0 0 +1 0 0 -1 -1 0 -1 -ROUND_START 315 0 0 0 0 1 -1 0 -1 SHOT_START 79 -ROUND_START 316 0 0 0 0 0 1 -1 -1 -ROUND_START 317 0 0 0 0 0 -1 -1 -1 -ROUND_START 318 -0 0 0 +1 0 0 -1 -1 -1 -ROUND_START 319 0 0 0 0 0 1 -1 -1 -SHOT_START 80 -ROUND_START 320 0 0 0 @@ -2970,16 +2638,11 @@ ROUND_START 320 0 0 1 -ROUND_START 321 -0 -0 -0 0 0 +SHOT_START 80 0 0 -1 -ROUND_START 322 0 0 1 @@ -2988,7 +2651,6 @@ ROUND_START 322 0 0 0 -ROUND_START 323 0 0 1 @@ -2997,45 +2659,37 @@ ROUND_START 323 0 0 0 -SHOT_START 81 -ROUND_START 324 -0 -0 0 0 1 -1 0 0 -ROUND_START 325 0 0 0 0 -1 +0 1 0 0 -ROUND_START 326 +0 +SHOT_START 81 0 0 0 0 1 1 -0 -0 -ROUND_START 327 +1 +1 0 0 0 0 1 1 -0 -0 -SHOT_START 82 -ROUND_START 328 +1 +1 0 0 0 @@ -3043,8 +2697,7 @@ ROUND_START 328 1 1 0 -0 -ROUND_START 329 +1 0 0 0 @@ -3052,8 +2705,9 @@ ROUND_START 329 1 1 0 +1 +SHOT_START 82 0 -ROUND_START 330 0 0 0 @@ -3062,7 +2716,6 @@ ROUND_START 330 1 0 0 -ROUND_START 331 0 0 0 @@ -3071,8 +2724,6 @@ ROUND_START 331 1 0 0 -SHOT_START 83 -ROUND_START 332 0 0 0 @@ -3081,7 +2732,6 @@ ROUND_START 332 1 0 0 -ROUND_START 333 0 0 0 @@ -3089,28 +2739,40 @@ ROUND_START 333 1 1 0 -0 -ROUND_START 334 +SHOT_START 83 0 0 0 0 1 1 +1 +1 0 0 -ROUND_START 335 0 0 +1 +1 +1 +1 +1 +1 0 0 1 1 +1 +1 +1 +1 0 +0 +1 +1 +1 1 SHOT_START 84 -ROUND_START 336 -0 0 0 0 @@ -3118,8 +2780,7 @@ ROUND_START 336 1 1 1 -ROUND_START 337 -0 +1 0 0 0 @@ -3127,8 +2788,7 @@ ROUND_START 337 1 1 1 -ROUND_START 338 -0 +1 0 0 0 @@ -3136,8 +2796,7 @@ ROUND_START 338 1 1 1 -ROUND_START 339 -0 +1 0 0 0 @@ -3145,17 +2804,13 @@ ROUND_START 339 1 1 1 +1 SHOT_START 85 -ROUND_START 340 -0 -0 -0 0 0 0 0 1 -ROUND_START 341 0 0 0 @@ -3164,7 +2819,6 @@ ROUND_START 341 0 0 1 -ROUND_START 342 0 0 0 @@ -3173,7 +2827,6 @@ ROUND_START 342 0 0 1 -ROUND_START 343 0 0 0 @@ -3182,120 +2835,109 @@ ROUND_START 343 0 0 1 -SHOT_START 86 -ROUND_START 344 0 0 0 +SHOT_START 86 0 0 -1 0 -1 -ROUND_START 345 0 +1 +1 +1 +1 0 0 0 0 1 -0 1 -ROUND_START 346 -0 +1 +1 0 0 0 0 1 -0 1 -ROUND_START 347 -0 +1 +1 0 0 0 0 1 -0 +1 +1 1 SHOT_START 87 -ROUND_START 348 0 0 0 0 1 0 -0 1 -ROUND_START 349 0 0 0 0 -1 0 +1 0 1 -ROUND_START 350 0 0 0 +1 0 1 0 +1 0 1 -ROUND_START 351 0 +1 0 0 0 1 0 -0 -1 SHOT_START 88 -ROUND_START 352 0 0 0 0 0 -1 0 1 -ROUND_START 353 -0 0 0 0 0 1 0 -1 -ROUND_START 354 0 +1 0 0 0 0 1 0 -1 -ROUND_START 355 0 +1 0 0 0 0 1 0 +0 1 -SHOT_START 89 -ROUND_START 356 0 +SHOT_START 89 0 0 0 @@ -3303,8 +2945,7 @@ ROUND_START 356 0 1 1 -ROUND_START 357 -0 +1 0 0 0 @@ -3312,63 +2953,58 @@ ROUND_START 357 0 1 1 -ROUND_START 358 -0 -0 -0 -0 -0 -0 1 1 -ROUND_START 359 0 0 0 +1 +1 +1 +1 +1 0 0 0 1 1 +1 +1 SHOT_START 90 -ROUND_START 360 0 0 0 0 0 1 -1 0 -ROUND_START 361 +1 0 0 0 0 0 1 -1 0 -ROUND_START 362 +1 0 0 0 0 0 1 -1 0 -ROUND_START 363 +1 0 0 0 0 0 1 -1 0 +1 SHOT_START 91 -ROUND_START 364 +0 0 0 0 @@ -3376,8 +3012,7 @@ ROUND_START 364 1 0 0 -1 -ROUND_START 365 +0 0 0 0 @@ -3385,8 +3020,7 @@ ROUND_START 365 1 0 0 -1 -ROUND_START 366 +0 0 0 0 @@ -3394,8 +3028,7 @@ ROUND_START 366 1 0 0 -1 -ROUND_START 367 +0 0 0 0 @@ -3403,157 +3036,139 @@ ROUND_START 367 1 0 0 -1 SHOT_START 92 -ROUND_START 368 0 0 0 0 -0 -1 1 1 -ROUND_START 369 0 +1 0 0 0 0 1 1 -1 -ROUND_START 370 0 +1 0 0 0 0 1 1 -1 -ROUND_START 371 0 +1 0 0 0 0 1 1 +0 1 SHOT_START 93 -ROUND_START 372 0 0 0 0 1 -0 1 0 -ROUND_START 373 -0 -0 -0 -0 1 -0 1 0 -ROUND_START 374 -0 -0 -0 -0 1 0 1 0 -ROUND_START 375 +1 +1 +1 0 +1 0 +1 0 +1 +1 +1 0 1 0 1 0 +1 +1 SHOT_START 94 -ROUND_START 376 0 0 0 0 -0 -1 1 +0 1 -ROUND_START 377 0 0 0 0 0 1 -1 -1 -ROUND_START 378 0 +1 0 0 0 +1 0 1 1 1 -ROUND_START 379 -0 -0 +1 0 0 +1 0 1 1 -0 +1 +1 SHOT_START 95 -ROUND_START 380 0 0 0 0 1 1 -0 -0 -ROUND_START 381 +1 +1 0 0 0 0 1 1 +1 +1 0 0 -ROUND_START 382 0 0 1 -0 +1 1 1 0 0 -ROUND_START 383 0 0 1 -0 1 1 -0 -0 +1 SHOT_START 96 -ROUND_START 384 0 0 0 @@ -3561,73 +3176,65 @@ ROUND_START 384 1 0 0 -1 -ROUND_START 385 0 0 0 +1 0 1 0 0 1 -ROUND_START 386 -0 0 0 +1 0 1 0 0 1 -ROUND_START 387 -0 0 0 +1 0 1 0 0 1 SHOT_START 97 -ROUND_START 388 0 0 0 0 -1 +0 +0 0 1 -1 -ROUND_START 389 0 0 0 0 -1 +0 +0 0 1 -1 -ROUND_START 390 0 0 0 0 -1 +0 +0 0 1 -1 -ROUND_START 391 0 0 0 0 -1 0 -1 +0 +0 1 SHOT_START 98 -ROUND_START 392 0 0 0 @@ -3636,7 +3243,6 @@ ROUND_START 392 0 0 1 -ROUND_START 393 0 0 0 @@ -3645,7 +3251,6 @@ ROUND_START 393 0 0 1 -ROUND_START 394 0 0 0 @@ -3654,7 +3259,6 @@ ROUND_START 394 0 0 1 -ROUND_START 395 0 0 0 @@ -3664,7 +3268,6 @@ ROUND_START 395 0 1 SHOT_START 99 -ROUND_START 396 0 0 0 @@ -3672,8 +3275,7 @@ ROUND_START 396 1 1 1 -0 -ROUND_START 397 +1 0 0 0 @@ -3681,8 +3283,7 @@ ROUND_START 397 1 1 1 -0 -ROUND_START 398 +1 0 0 0 @@ -3690,8 +3291,7 @@ ROUND_START 398 1 1 1 -0 -ROUND_START 399 +1 0 0 0 @@ -3699,7 +3299,7 @@ ROUND_START 399 1 1 1 -0 +1 CORRECTIONS_START 0 0 @@ -3707,22 +3307,16 @@ CORRECTIONS_START 0 0 0 +1 0 0 0 0 0 -0 -1 -0 1 0 0 0 -1 -0 -1 -0 0 0 0 @@ -3746,6 +3340,7 @@ CORRECTIONS_START 0 0 0 +1 0 0 0 @@ -3755,6 +3350,7 @@ CORRECTIONS_START 0 0 0 +1 0 0 0 @@ -3765,12 +3361,12 @@ CORRECTIONS_START 0 0 0 -1 0 0 0 0 0 +1 0 0 0 @@ -3792,9 +3388,13 @@ CORRECTIONS_START 0 0 0 +1 +1 +1 0 0 0 +1 0 0 0 diff --git a/libs/qec/unittests/realtime/test_decoding_server.cpp b/libs/qec/unittests/realtime/test_decoding_server.cpp deleted file mode 100644 index 9ab8b50eb..000000000 --- a/libs/qec/unittests/realtime/test_decoding_server.cpp +++ /dev/null @@ -1,482 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - ******************************************************************************/ - -/// Two-process realtime decoding-server test: this process runs only the -/// caller (simulated QPU) side; the decoder lives in a separate -/// decoding_server process reached through a CUDA-Q device_call -/// channel. Both the decoder and the transport are configuration: -/// - the decoder comes from the YAML config file handed to the server -/// (swapping decoders is a config-file change, not a code change); -/// - the transport defaults to `udp` (loopback; runs anywhere) and can be -/// switched to the CPU RoCE RDMA wire with -/// QEC_DECODING_SERVER_TRANSPORT=cpu_roce -/// plus the RDMA topology env vars shared with CUDA-Q's -/// CpuRoceChannelTester: -/// CUDAQ_CPU_ROCE_TEST_CHANNEL_DEVICE / CUDAQ_CPU_ROCE_TEST_CHANNEL_IP -/// CUDAQ_CPU_ROCE_TEST_DAEMON_DEVICE / CUDAQ_CPU_ROCE_TEST_DAEMON_IP -/// -/// The server is spawned as a subprocess, its ephemeral port read from the -/// QEC_DECODING_SERVER_READY stdout line, and its dispatch count (printed at -/// shutdown) is the proof the device_calls crossed the process boundary -- -/// there is no decoder configured in this process at all. -/// -/// The kernel's block/syndrome size and expected correction must stay -/// consistent with the H/O/D matrices in the config file (3-bit identity: -/// syndrome bit 1 set -> correction bit 1 set for any sane decoder). - -#include "CqrTransceiver.h" -#include "cudaq.h" -#include "cudaq/qec/realtime/decoding.h" -#include "cudaq/realtime.h" -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace { - -constexpr std::uint64_t kRunShots = 1; -constexpr std::int64_t kExpectedCorrection = 1; - -TEST(CqrTransceiverTest, RejectsPayloadBeyondReportedSlot) { - using namespace cudaq::qec::decoding_server; - - // 4-arg wire format: [u64 decoder_id][u64 counter][u64 mapping_id] - // [u64 num_syndromes (the vector element-count prefix)] - // [u8 x ceil(bits/8) bit-packed syndromes] - constexpr std::size_t payload_size = 4 * sizeof(uint64_t) + 1; - std::array rx{}; - std::array tx{}; - cudaq::realtime::RPCHeader header{}; - header.magic = cudaq::realtime::RPC_MAGIC_REQUEST; - header.function_id = kEnqueueSyndromesFunctionId; - header.arg_len = payload_size; - header.request_id = 17; - std::memcpy(rx.data(), &header, sizeof(header)); - - // Keep accessible, valid-looking payload bytes beyond the reported slot. - // A parser that trusts arg_len instead of slot_size will incorrectly accept - // this request even without a sanitizer detecting the contract violation. - const std::array fields = {/*decoder_id=*/3, /*counter=*/7, - /*mapping_id=*/0, - /*num_syndromes=*/1}; - std::memcpy(rx.data() + sizeof(header), fields.data(), sizeof(fields)); - rx.back() = 1; // bit-packed syndrome byte - - CqrTransceiver transceiver; - transceiver.inject(rx.data(), tx.data(), sizeof(header), header.function_id); - - const auto *response = - reinterpret_cast(tx.data()); - EXPECT_EQ(response->magic, cudaq::realtime::RPC_MAGIC_RESPONSE); - EXPECT_EQ(response->status, static_cast(RpcStatus::BAD_REQUEST)); - EXPECT_EQ(response->request_id, header.request_id); -} - -TEST(CqrTransceiverTest, AcceptsAnExactlySizedEnqueuePayload) { - using namespace cudaq::qec::decoding_server; - - // 4-arg wire format (see RejectsPayloadBeyondReportedSlot). - constexpr std::size_t payload_size = 4 * sizeof(uint64_t) + 1; - std::array rx{}; - std::array tx{}; - cudaq::realtime::RPCHeader header{}; - header.magic = cudaq::realtime::RPC_MAGIC_REQUEST; - header.function_id = kEnqueueSyndromesFunctionId; - header.arg_len = payload_size; - header.request_id = 23; - std::memcpy(rx.data(), &header, sizeof(header)); - - const std::array fields = {/*decoder_id=*/3, /*counter=*/7, - /*mapping_id=*/0, - /*num_syndromes=*/1}; - std::memcpy(rx.data() + sizeof(header), fields.data(), sizeof(fields)); - rx.back() = 1; // bit-packed syndrome byte - - CqrTransceiver transceiver; - transceiver.inject(rx.data(), tx.data(), rx.size(), header.function_id); - auto frame = transceiver.recv(); - - ASSERT_EQ(frame.buf.size(), sizeof(RPCHeader) + sizeof(EnqueuePayload) + 1); - const auto *request = reinterpret_cast( - frame.buf.data() + sizeof(RPCHeader)); - EXPECT_EQ(request->decoder_id, 3); - EXPECT_EQ(request->counter, 7); - EXPECT_EQ(request->syndrome_mapping_id, 0); - EXPECT_EQ(request->num_syndromes, 1); - EXPECT_EQ(frame.buf.back(), 1); - - const auto *response = - reinterpret_cast(tx.data()); - EXPECT_EQ(response->magic, cudaq::realtime::RPC_MAGIC_RESPONSE); - EXPECT_EQ(response->status, 0); -} - -__qpu__ std::int64_t decoding_server_kernel() { - constexpr std::uint64_t kKernelDecoderId = 0; - constexpr std::uint64_t kKernelBlockSize = 3; - constexpr std::uint64_t kKernelSyndromeSize = 3; - constexpr std::uint64_t kKernelSyndromeTag = 1; - constexpr std::size_t kKernelActiveSyndromeIndex = 1; - - cudaq::qec::decoding::reset_decoder(/*decoder_id=*/kKernelDecoderId); - - std::vector syndrome(kKernelSyndromeSize); - for (std::size_t i = 0; i < kKernelSyndromeSize; ++i) - syndrome[i] = false; - syndrome[kKernelActiveSyndromeIndex] = true; - cudaq::qec::decoding::enqueue_syndromes_test( - /*decoder_id=*/kKernelDecoderId, syndrome, /*tag=*/kKernelSyndromeTag); - - auto corrections = cudaq::qec::decoding::get_corrections( - /*decoder_id=*/kKernelDecoderId, /*return_size=*/kKernelBlockSize, - /*reset=*/true); - return corrections[kKernelActiveSyndromeIndex] ? std::int64_t{1} - : std::int64_t{0}; -} - -std::string env_or(const char *name, const std::string &fallback) { - const char *value = std::getenv(name); - return value && *value ? std::string(value) : fallback; -} - -// The server binary path is baked in at configure time (the server is built -// from libs/qec/tools/decoding-server); QEC_DECODING_SERVER overrides it. The -// example decoder configs are placed in the same directory as the server. -std::string server_path() { - return env_or("QEC_DECODING_SERVER", QEC_DECODING_SERVER_PATH); -} - -std::string server_dir() { - const std::string path = server_path(); - const auto slash = path.find_last_of('/'); - return slash == std::string::npos ? "." : path.substr(0, slash); -} - -// Spawns decoding_server with the given decoder config file, hands back -// its READY port (udp: the UDP data port; cpu_roce: the TCP rendezvous port), -// and collects its stdout (for the shutdown dispatch-count line). -class ServerProcess { -public: - bool start(const std::string &config_file, std::string &error, - int ready_timeout_ms = 15000) { - int out_pipe[2] = {-1, -1}; - if (::pipe(out_pipe) != 0) { - error = "pipe() failed"; - return false; - } - pid = ::fork(); - if (pid < 0) { - error = "fork() failed"; - return false; - } - if (pid == 0) { - ::dup2(out_pipe[1], STDOUT_FILENO); - ::close(out_pipe[0]); - ::close(out_pipe[1]); - const std::string server = server_path(); - const std::string config_arg = - "--config=" + (!config_file.empty() && config_file[0] == '/' - ? config_file - : server_dir() + "/" + config_file); - const std::string transport_arg = - "--transport=" + env_or("QEC_DECODING_SERVER_TRANSPORT", "udp"); - const std::string device_arg = - "--device=" + env_or("CUDAQ_CPU_ROCE_TEST_DAEMON_DEVICE", "mlx5_0"); - const std::string local_ip_arg = - "--local-ip=" + env_or("CUDAQ_CPU_ROCE_TEST_DAEMON_IP", "10.0.0.2"); - ::execl(server.c_str(), server.c_str(), config_arg.c_str(), - transport_arg.c_str(), device_arg.c_str(), local_ip_arg.c_str(), - "--port=0", "--timeout=60", static_cast(nullptr)); - std::perror("execl decoding_server"); - _exit(127); - } - ::close(out_pipe[1]); - outFd = out_pipe[0]; - - // Read stdout until the READY line (or the deadline; decoders that - // build TensorRT engines at startup need more than the default 15 s). - std::string ready_line; - if (!readLineWithPrefix("QEC_DECODING_SERVER_READY", ready_timeout_ms, - ready_line)) { - error = "server did not print QEC_DECODING_SERVER_READY; output so " - "far: " + - captured; - return false; - } - if (std::sscanf(ready_line.c_str(), "QEC_DECODING_SERVER_READY port=%hu", - &port) != 1) { - error = "could not parse server port from: " + ready_line; - return false; - } - return true; - } - - // Terminate the server and return its dispatched-request count (-1 if the - // shutdown line never appeared). Also captures the per-decoder-worker - // concurrency high-water mark into max_concurrent_decoders. - std::int64_t stopAndGetDispatchCount() { - if (pid <= 0) - return -1; - ::kill(pid, SIGTERM); - std::string line; - std::int64_t count = -1; - if (readLineWithPrefix("QEC_DECODING_SERVER_DISPATCHED", 10000, line)) { - long long parsed = -1; - if (std::sscanf(line.c_str(), "QEC_DECODING_SERVER_DISPATCHED count=%lld", - &parsed) == 1) - count = parsed; - } - if (readLineWithPrefix("QEC_DECODING_SERVER_MAX_CONCURRENT_DECODERS", 5000, - line)) { - long long parsed = -1; - if (std::sscanf(line.c_str(), - "QEC_DECODING_SERVER_MAX_CONCURRENT_DECODERS count=%lld", - &parsed) == 1) - max_concurrent_decoders = parsed; - } - int status = 0; - ::waitpid(pid, &status, 0); - pid = -1; - if (outFd >= 0) { - ::close(outFd); - outFd = -1; - } - return count; - } - - ~ServerProcess() { - if (pid > 0) { - ::kill(pid, SIGKILL); - int status = 0; - ::waitpid(pid, &status, 0); - } - if (outFd >= 0) - ::close(outFd); - } - - std::uint16_t port = 0; - std::string captured; - std::int64_t max_concurrent_decoders = -1; - -private: - bool readLineWithPrefix(const char *prefix, int timeout_ms, - std::string &line_out) { - std::string line; - const auto deadline_ms = timeout_ms; - int waited_ms = 0; - while (waited_ms < deadline_ms) { - pollfd pfd{outFd, POLLIN, 0}; - const int ready = ::poll(&pfd, 1, 100); - waited_ms += 100; - if (ready <= 0) - continue; - char c = 0; - while (::read(outFd, &c, 1) == 1) { - captured.push_back(c); - if (c == '\n') { - if (line.rfind(prefix, 0) == 0) { - line_out = line; - return true; - } - line.clear(); - } else { - line.push_back(c); - } - pollfd inner{outFd, POLLIN, 0}; - if (::poll(&inner, 1, 0) <= 0) - break; - } - } - return false; - } - - pid_t pid = -1; - int outFd = -1; -}; - -struct RealtimeGuard { - bool armed = false; - ~RealtimeGuard() { - if (armed) - cudaq::realtime::finalize(); - } -}; - -} // namespace - -// Caller-side device_call channel arguments for the selected transport. -std::vector channel_arguments(std::uint16_t server_port) { - const std::string transport = env_or("QEC_DECODING_SERVER_TRANSPORT", "udp"); - if (transport == "cpu_roce") { - // The server's READY port is its TCP rendezvous port; the RDMA topology - // comes from the same env vars CUDA-Q's CpuRoceChannelTester uses. - // Unlike udp, the RDMA ring geometry is part of the wire contract: the - // channel writes requests directly into the server's rings, so slots / - // slot-size must match the server's --num-slots / --slot-size defaults - // (8 x 256, see decoding_server.cpp ServerConfig). - return {"--cudaq-device-call=cpu_roce", - "--cudaq-device-call-slots=8", - "--cudaq-device-call-slot-size=256", - "ib-device=" + - env_or("CUDAQ_CPU_ROCE_TEST_CHANNEL_DEVICE", "mlx5_0"), - "local-ip=" + env_or("CUDAQ_CPU_ROCE_TEST_CHANNEL_IP", "10.0.0.1"), - "rendezvous-host=" + - env_or("CUDAQ_CPU_ROCE_TEST_DAEMON_IP", "10.0.0.2"), - "rendezvous-port=" + std::to_string(server_port)}; - } - return {"--cudaq-device-call=udp", "udp-host=127.0.0.1", - "udp-port=" + std::to_string(server_port)}; -} - -// Runs the full two-process round-trip against a server configured with -// `config_file`. Each invocation spawns a fresh server on an ephemeral port. -void run_two_process_decode_test(const std::string &config_file) { - ServerProcess server; - std::string error; - ASSERT_TRUE(server.start(config_file, error)) << error; - - std::vector args = {"test_decoding_server"}; - for (auto &arg : channel_arguments(server.port)) - args.push_back(std::move(arg)); - std::vector argv; - for (auto &arg : args) - argv.push_back(arg.data()); - argv.push_back(nullptr); - int argc = static_cast(args.size()); - cudaq::realtime::initialize(argc, argv.data()); - RealtimeGuard realtime_guard{true}; - - const auto results = cudaq::run(kRunShots, decoding_server_kernel); - ASSERT_EQ(results.size(), kRunShots); - EXPECT_EQ(results[0], kExpectedCorrection); - - // Two-process self-verification: the decode can only have happened in the - // server (this process configured no decoders), and the server's dispatch - // counter proves the device_calls crossed the selected transport. Three - // calls: reset_decoder, enqueue_syndromes, get_corrections. - const std::int64_t dispatched = server.stopAndGetDispatchCount(); - EXPECT_GE(dispatched, 3) << "server output:\n" << server.captured; -} - -TEST(DecodingServerTwoProcess, TwoProcessHostDispatch) { - run_two_process_decode_test("decoding_server_config.yaml"); -} - -TEST(DecodingServerTwoProcess, TwoProcessHostDispatchMultiErrorLut) { - run_two_process_decode_test("decoding_server_config_multi_error_lut.yaml"); -} - -// --------------------------------------------------------------------------- -// Two decoders (two logical qubits) in ONE server, driven from ONE __qpu__ -// kernel: qubit A uses decoder 0 and qubit B uses decoder 1, with different -// active syndrome bits. On the server each decoder executes on its own -// per-decoder worker thread; the server's shutdown -// QEC_DECODING_SERVER_MAX_CONCURRENT_DECODERS line reports the busy -// high-water mark of those workers. -// --------------------------------------------------------------------------- - -__qpu__ std::int64_t dual_decoding_server_kernel() { - constexpr std::uint64_t kDecoderA = 0; - constexpr std::uint64_t kDecoderB = 1; - constexpr std::uint64_t kBlockSize = 3; - constexpr std::uint64_t kSyndromeSize = 3; - constexpr std::size_t kActiveA = 1; - constexpr std::size_t kActiveB = 2; - - cudaq::qec::decoding::reset_decoder(kDecoderA); - cudaq::qec::decoding::reset_decoder(kDecoderB); - - std::vector syndrome_a(kSyndromeSize); - std::vector syndrome_b(kSyndromeSize); - for (std::size_t i = 0; i < kSyndromeSize; ++i) { - syndrome_a[i] = false; - syndrome_b[i] = false; - } - syndrome_a[kActiveA] = true; - syndrome_b[kActiveB] = true; - cudaq::qec::decoding::enqueue_syndromes_test(kDecoderA, syndrome_a, - /*tag=*/1); - cudaq::qec::decoding::enqueue_syndromes_test(kDecoderB, syndrome_b, - /*tag=*/1); - - auto corr_a = cudaq::qec::decoding::get_corrections(kDecoderA, kBlockSize, - /*reset=*/true); - auto corr_b = cudaq::qec::decoding::get_corrections(kDecoderB, kBlockSize, - /*reset=*/true); - std::int64_t out = 0; - if (corr_a[kActiveA]) - out = out + 1; // bit 0: decoder 0 corrected its active bit - if (corr_b[kActiveB]) - out = out + 2; // bit 1: decoder 1 corrected its active bit - if (corr_a[kActiveB]) - out = out + 4; // bit 2 set = cross-talk (decoder 0 saw B's syndrome) - if (corr_b[kActiveA]) - out = out + 8; // bit 3 set = cross-talk (decoder 1 saw A's syndrome) - return out; -} - -TEST(DecodingServerTwoProcess, TwoProcessHostDispatchDualDecoders) { - // Two identical 3-bit-identity pymatching decoders (ids 0 and 1) in one - // server -- one per logical qubit. - const std::string config_path = - ::testing::TempDir() + "/decoding_server_dual_config.yaml"; - { - std::ofstream config_file(config_path); - config_file << "decoders:\n"; - for (int id = 0; id < 2; ++id) { - config_file << " - id: " << id << "\n" - << " type: pymatching\n" - << " block_size: 3\n" - << " syndrome_size: 3\n" - << " H_sparse: [0, -1, 1, -1, 2, -1]\n" - << " O_sparse: [0, -1, 1, -1, 2, -1]\n" - << " D_sparse: [0, -1, 1, -1, 2, -1]\n" - << " decoder_custom_args:\n" - << " merge_strategy: smallest_weight\n" - << " error_rate_vec: [0.1, 0.1, 0.1]\n"; - } - } - - ServerProcess server; - std::string error; - ASSERT_TRUE(server.start(config_path, error)) << error; - - std::vector args = {"test_decoding_server"}; - for (auto &arg : channel_arguments(server.port)) - args.push_back(std::move(arg)); - std::vector argv; - for (auto &arg : args) - argv.push_back(arg.data()); - argv.push_back(nullptr); - int argc = static_cast(args.size()); - cudaq::realtime::initialize(argc, argv.data()); - RealtimeGuard realtime_guard{true}; - - const auto results = cudaq::run(kRunShots, dual_decoding_server_kernel); - ASSERT_EQ(results.size(), kRunShots); - EXPECT_EQ(results[0], 3); - - // Six calls crossed the wire (reset/enqueue/get per decoder), and both - // decoders' execution workers ran (high-water mark >= 1; == 2 when the - // decodes genuinely overlapped, which tiny identity decodes need not). - const std::int64_t dispatched = server.stopAndGetDispatchCount(); - EXPECT_GE(dispatched, 6) << "server output:\n" << server.captured; - EXPECT_GE(server.max_concurrent_decoders, 1) << "server output:\n" - << server.captured; -} diff --git a/libs/qec/unittests/realtime/test_realtime_predecoder_w_pymatching.cpp b/libs/qec/unittests/realtime/test_realtime_predecoder_w_pymatching.cpp index 679ec6d50..827ce4649 100644 --- a/libs/qec/unittests/realtime/test_realtime_predecoder_w_pymatching.cpp +++ b/libs/qec/unittests/realtime/test_realtime_predecoder_w_pymatching.cpp @@ -32,7 +32,7 @@ #include #include "cudaq/qec/realtime/ai_decoder_service.h" -#include "cudaq/realtime/daemon/dispatcher/graph_launch_engine.h" +#include "cudaq/realtime/daemon/dispatcher/host_dispatcher.h" #define CUDA_CHECK(call) \ do { \ diff --git a/libs/qec/unittests/test_decoders_yaml.cpp b/libs/qec/unittests/test_decoders_yaml.cpp index 01d8878cd..1b3b0178a 100644 --- a/libs/qec/unittests/test_decoders_yaml.cpp +++ b/libs/qec/unittests/test_decoders_yaml.cpp @@ -6,10 +6,8 @@ * the terms of the Apache License 2.0 which accompanies this distribution. * ******************************************************************************/ -#include "SessionRegistry.h" #include "../lib/realtime/realtime_decoding.h" #include "cudaq/qec/decoder.h" -#include "cudaq/qec/decoder_config_schema.h" #include "cudaq/qec/pcm_utils.h" #include "cudaq/qec/realtime/decoding_config.h" #include @@ -18,9 +16,7 @@ #include #include #include -#include #include -#include namespace { class ScopedEnv { @@ -51,46 +47,6 @@ void get_corrections(std::uint64_t decoder_id, uint8_t *corrections, std::uint64_t correction_length, bool reset); } // namespace cudaq::qec::decoding::simulation -TEST(DecoderYAMLTest, RejectsParserErrors) { - const std::string unknown_root_key = R"( -decoders: - - id: 0 - type: pymatching - block_size: 1 - syndrome_size: 1 - H_sparse: [0, -1] - O_sparse: [0, -1] - D_sparse: [0, -1] -unexpected: true -)"; - EXPECT_THROW( - cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( - unknown_root_key), - std::runtime_error); - - const std::string misspelled_decoder_argument = R"( -decoders: - - id: 0 - type: pymatching - block_size: 1 - syndrome_size: 1 - H_sparse: [0, -1] - O_sparse: [0, -1] - D_sparse: [0, -1] - decoder_custom_args: - merge_stratgey: smallest_weight -)"; - EXPECT_THROW( - cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( - misspelled_decoder_argument), - std::runtime_error); - - EXPECT_THROW( - cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( - "decoders: ["), - std::runtime_error); -} - /// Helper function to test that a decoder configuration can be serialized to /// and from YAML. void test_decoder_yaml_roundtrip( @@ -144,17 +100,6 @@ create_test_empty_decoder_config(int id) { return config; } -cudaq::qec::decoding::config::decoder_config -create_test_sample_realtime_decoder_config(int id) { - auto config = create_test_empty_decoder_config(id); - config.type = "sample_decoder"; - cudaqx::tensor O({2, config.block_size}); - O.at({0, 0}) = 1; - O.at({1, 1}) = 1; - config.O_sparse = cudaq::qec::pcm_to_sparse_vec(O); - return config; -} - /// Helper function to create a sample, skeleton test decoder configuration for /// the NV-QLDPC decoder. cudaq::qec::decoding::config::decoder_config @@ -163,48 +108,43 @@ create_test_decoder_config_nv_qldpc(int id) { create_test_empty_decoder_config(id); config.type = "nv-qldpc-decoder"; - cudaqx::heterogeneous_map nv_args; - nv_args.insert("use_sparsity", true); - nv_args.insert("max_iterations", 50); - nv_args.insert("use_osd", true); - nv_args.insert("osd_order", 60); - nv_args.insert("osd_method", 3); - nv_args.insert("error_rate_vec", std::vector(config.block_size, 0.1)); - nv_args.insert("n_threads", 128); - nv_args.insert("bp_batch_size", 1); - nv_args.insert("osd_batch_size", 16); - nv_args.insert("iter_per_check", 2); - nv_args.insert("clip_value", 10.0); - nv_args.insert("bp_method", 3); - nv_args.insert("scale_factor", 1.0); - nv_args.insert("proc_float", "fp64"); - nv_args.insert("gamma0", 0.0); - nv_args.insert("gamma_dist", std::vector{0.1, 0.2}); - cudaqx::heterogeneous_map srelay_args; - srelay_args.insert("pre_iter", std::size_t{5}); - srelay_args.insert("num_sets", std::size_t{10}); - srelay_args.insert("stopping_criterion", "NConv"); - srelay_args.insert("stop_nconv", std::size_t{10}); - nv_args.insert("srelay_config", srelay_args); + config.decoder_custom_args = + cudaq::qec::decoding::config::nv_qldpc_decoder_config(); + auto &nv_config = + std::get( + config.decoder_custom_args); + nv_config.use_sparsity = true; + nv_config.max_iterations = 50; + nv_config.use_osd = true; + nv_config.osd_order = 60; + nv_config.osd_method = 3; + nv_config.error_rate_vec = + std::vector(config.block_size, 0.1); + + nv_config.n_threads = 128; + nv_config.bp_batch_size = 1; + nv_config.osd_batch_size = 16; + nv_config.iter_per_check = 2; + nv_config.clip_value = 10.0; + nv_config.bp_method = 3; + nv_config.scale_factor = 1.0; + nv_config.proc_float = "fp64"; + nv_config.gamma0 = 0.0; + nv_config.gamma_dist = {0.1, 0.2}; + nv_config.srelay_config = cudaq::qec::decoding::config::srelay_bp_config(); + nv_config.srelay_config->pre_iter = 5; + nv_config.srelay_config->num_sets = 10; + nv_config.srelay_config->stopping_criterion = "NConv"; + nv_config.srelay_config->stop_nconv = 10; // explicit_gammas must have num_sets rows (10 in this case) - nv_args.insert("explicit_gammas", - std::vector>( - 10, std::vector(config.block_size, 0.1))); - nv_args.insert("bp_seed", 42); - nv_args.insert("composition", 1); - config.decoder_custom_args = nv_args; + nv_config.explicit_gammas = std::vector>( + 10, std::vector(config.block_size, 0.1)); + nv_config.bp_seed = 42; + nv_config.composition = 1; return config; } -// The trt_decoder schema is registered by the trt_decoder plugin, which is -// only built when TensorRT is available. YAML paths for trt configs require -// it; typed-struct conversions do not. -bool is_trt_decoder_schema_available() { - return cudaq::qec::decoding::config::find_decoder_schema("trt_decoder") != - nullptr; -} - bool is_nv_qldpc_decoder_available() { try { std::size_t block_size = 7; @@ -258,9 +198,12 @@ TEST(DecoderYAMLTest, MultiLUTDecoder) { cudaq::qec::decoding::config::decoder_config config = create_test_empty_decoder_config(0); config.type = "multi_error_lut"; - cudaqx::heterogeneous_map lut_args; - lut_args.insert("lut_error_depth", 2); - config.decoder_custom_args = lut_args; + config.decoder_custom_args = + cudaq::qec::decoding::config::multi_error_lut_config(); + auto &lut_config = + std::get( + config.decoder_custom_args); + lut_config.lut_error_depth = 2; multi_config.decoders.push_back(config); test_decoder_yaml_roundtrip(multi_config); @@ -272,7 +215,8 @@ TEST(DecoderYAMLTest, SingleLUTDecoder) { cudaq::qec::decoding::config::decoder_config config = create_test_empty_decoder_config(0); config.type = "single_error_lut"; - config.decoder_custom_args = cudaqx::heterogeneous_map(); + config.decoder_custom_args = + cudaq::qec::decoding::config::single_error_lut_config(); multi_config.decoders.push_back(config); test_decoder_yaml_roundtrip(multi_config); @@ -290,36 +234,37 @@ create_test_decoder_config_trt(int id) { O.at({1, 3}) = 1; config.O_sparse = cudaq::qec::pcm_to_sparse_vec(O); - cudaqx::heterogeneous_map trt_args; - trt_args.insert("onnx_load_path", "/tmp/predecoder.onnx"); - trt_args.insert("engine_save_path", "/tmp/predecoder.engine"); - trt_args.insert("precision", "best"); - trt_args.insert("memory_workspace", std::size_t{1ULL << 20}); - trt_args.insert("batch_size", std::size_t{4}); - trt_args.insert("use_cuda_graph", false); - trt_args.insert("global_decoder", "pymatching"); - cudaqx::heterogeneous_map pymatching_params; - pymatching_params.insert("merge_strategy", "smallest_weight"); - pymatching_params.insert("error_rate_vec", - std::vector(config.block_size, 0.1)); - trt_args.insert("global_decoder_params", pymatching_params); - config.decoder_custom_args = trt_args; + config.decoder_custom_args = + cudaq::qec::decoding::config::trt_decoder_config(); + auto &trt_config = std::get( + config.decoder_custom_args); + trt_config.onnx_load_path = "/tmp/predecoder.onnx"; + trt_config.engine_save_path = "/tmp/predecoder.engine"; + trt_config.precision = "best"; + trt_config.memory_workspace = 1ULL << 20; + trt_config.batch_size = 4; + trt_config.use_cuda_graph = false; + trt_config.global_decoder = "pymatching"; + auto pymatching_params = cudaq::qec::decoding::config::pymatching_config(); + pymatching_params.merge_strategy = "smallest_weight"; + pymatching_params.error_rate_vec = + std::vector(config.block_size, 0.1); + trt_config.global_decoder_params = pymatching_params; return config; } TEST(DecoderYAMLTest, TrtDecoderConfigRoundTrip) { - if (!is_trt_decoder_schema_available()) - GTEST_SKIP() << "trt_decoder plugin (and its parameter schema) not built"; cudaq::qec::decoding::config::multi_decoder_config multi_config; multi_config.decoders.push_back(create_test_decoder_config_trt(0)); test_decoder_yaml_roundtrip(multi_config); - const auto &args = multi_config.decoders[0].decoder_custom_args.map(); - ASSERT_TRUE(args.contains("global_decoder_params")); - EXPECT_EQ(args.get("global_decoder_params") - .get("merge_strategy"), - "smallest_weight"); + const auto &trt_config = + std::get( + multi_config.decoders[0].decoder_custom_args); + EXPECT_TRUE( + std::holds_alternative( + trt_config.global_decoder_params)); } TEST(DecoderYAMLTest, TrtDecoderConfigToHeterogeneousMap) { @@ -360,15 +305,12 @@ TEST(DecoderYAMLTest, TrtDecoderRealtimeParamsIncludeObservableMatrix) { EXPECT_EQ(global_O.shape()[1], config.block_size); } -TEST(DecoderYAMLTest, TrtDecoderEmptyGlobalDecoderParams) { - if (!is_trt_decoder_schema_available()) - GTEST_SKIP() << "trt_decoder plugin (and its parameter schema) not built"; - // An explicitly empty global params section round-trips and reaches the - // realtime decoder params. +TEST(DecoderYAMLTest, TrtDecoderMonostateGlobalDecoderParams) { auto config = create_test_decoder_config_trt(0); - auto args = config.decoder_custom_args.map(); - args.insert("global_decoder_params", cudaqx::heterogeneous_map()); - config.decoder_custom_args = args; + auto &trt_config = std::get( + config.decoder_custom_args); + trt_config.global_decoder = "pymatching"; + trt_config.global_decoder_params = std::monostate{}; auto params = config.decoder_custom_args_to_heterogeneous_map(); EXPECT_TRUE(params.contains("global_decoder_params")); @@ -382,12 +324,12 @@ TEST(DecoderYAMLTest, TrtDecoderEmptyGlobalDecoderParams) { auto round_tripped = cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str(yaml); EXPECT_EQ(round_tripped.to_yaml_str(200), yaml); - const auto &round_tripped_args = - round_tripped.decoders[0].decoder_custom_args.map(); - ASSERT_TRUE(round_tripped_args.contains("global_decoder_params")); + const auto &round_tripped_trt_config = + std::get( + round_tripped.decoders[0].decoder_custom_args); EXPECT_TRUE( - round_tripped_args.get("global_decoder_params") - .empty()); + std::holds_alternative( + round_tripped_trt_config.global_decoder_params)); params = cudaq::qec::decoding::host::prepare_decoder_params(config); EXPECT_TRUE(params.contains("global_decoder_params")); @@ -400,10 +342,20 @@ TEST(DecoderYAMLTest, TrtDecoderEmptyGlobalDecoderParams) { } TEST(DecoderYAMLTest, TrtDecoderDefaultGlobalDecoderParams) { - if (!is_trt_decoder_schema_available()) - GTEST_SKIP() << "trt_decoder plugin (and its parameter schema) not built"; - // When the YAML names a global decoder with a registered schema but gives - // no params, an empty section is materialized on parse. + cudaqx::heterogeneous_map map; + map.insert("global_decoder", std::string("chromobius")); + + auto trt_config = + cudaq::qec::decoding::config::trt_decoder_config::from_heterogeneous_map( + map); + EXPECT_TRUE( + std::holds_alternative( + trt_config.global_decoder_params)); + auto params = trt_config.to_heterogeneous_map(); + EXPECT_TRUE(params.contains("global_decoder_params")); + EXPECT_TRUE( + params.get("global_decoder_params").empty()); + const std::string yaml_without_params = R"( decoders: - id: 0 @@ -416,27 +368,59 @@ TEST(DecoderYAMLTest, TrtDecoderDefaultGlobalDecoderParams) { decoder_custom_args: global_decoder: chromobius )"; - auto parsed = + auto parsed_without_params = cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( yaml_without_params); - const auto &args = parsed.decoders[0].decoder_custom_args.map(); - ASSERT_TRUE(args.contains("global_decoder_params")); + const auto &parsed_trt_config = + std::get( + parsed_without_params.decoders[0].decoder_custom_args); EXPECT_TRUE( - args.get("global_decoder_params").empty()); + std::holds_alternative( + parsed_trt_config.global_decoder_params)); - // Emission after materialization is stable. - const auto emitted = parsed.to_yaml_str(200); - EXPECT_NE(emitted.find("global_decoder_params"), std::string::npos); + auto config = create_test_decoder_config_trt(0); + auto &yaml_trt_config = + std::get( + config.decoder_custom_args); + yaml_trt_config.global_decoder = "chromobius"; + yaml_trt_config.global_decoder_params = std::monostate{}; + cudaq::qec::decoding::config::multi_decoder_config multi_config; + multi_config.decoders.push_back(config); + const auto yaml = multi_config.to_yaml_str(200); + EXPECT_NE(yaml.find("global_decoder_params"), std::string::npos); auto round_tripped = - cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( - emitted); - EXPECT_EQ(round_tripped, parsed); - EXPECT_EQ(round_tripped.to_yaml_str(200), emitted); + cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str(yaml); + const auto &round_tripped_trt_config = + std::get( + round_tripped.decoders[0].decoder_custom_args); + EXPECT_TRUE( + std::holds_alternative( + round_tripped_trt_config.global_decoder_params)); } TEST(DecoderYAMLTest, UnknownTrtGlobalDecoderParamsThrow) { - if (!is_trt_decoder_schema_available()) - GTEST_SKIP() << "trt_decoder plugin (and its parameter schema) not built"; + cudaqx::heterogeneous_map map; + map.insert("global_decoder", std::string("my_plugin")); + map.insert("global_decoder_params", cudaqx::heterogeneous_map{}); + EXPECT_THROW( + cudaq::qec::decoding::config::trt_decoder_config::from_heterogeneous_map( + map), + std::runtime_error); + + cudaq::qec::decoding::config::trt_decoder_config trt_config; + trt_config.global_decoder = "my_plugin"; + auto params = trt_config.to_heterogeneous_map(); + EXPECT_EQ(params.get("global_decoder"), "my_plugin"); + EXPECT_FALSE(params.contains("global_decoder_params")); + + map = cudaqx::heterogeneous_map(); + map.insert("global_decoder", std::string("my_plugin")); + trt_config = + cudaq::qec::decoding::config::trt_decoder_config::from_heterogeneous_map( + map); + EXPECT_TRUE( + std::holds_alternative(trt_config.global_decoder_params)); + const std::string yaml_with_unknown_params = R"( decoders: - id: 0 @@ -454,50 +438,25 @@ TEST(DecoderYAMLTest, UnknownTrtGlobalDecoderParamsThrow) { cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( yaml_with_unknown_params), std::runtime_error); - - // A global decoder without a registered schema is allowed as long as no - // params section is given (nothing is materialized for it). - const std::string yaml_without_params = R"( -decoders: - - id: 0 - type: trt_decoder - block_size: 1 - syndrome_size: 1 - H_sparse: [0, -1] - O_sparse: [] - D_sparse: [0, -1] - decoder_custom_args: - global_decoder: my_plugin -)"; - auto parsed = - cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( - yaml_without_params); - const auto &args = parsed.decoders[0].decoder_custom_args.map(); - EXPECT_EQ(args.get("global_decoder"), "my_plugin"); - EXPECT_FALSE(args.contains("global_decoder_params")); } TEST(DecoderYAMLTest, TrtDecoderParamsWithoutDecoderThrows) { - if (!is_trt_decoder_schema_available()) - GTEST_SKIP() << "trt_decoder plugin (and its parameter schema) not built"; - const std::string yaml_params_without_decoder = R"( -decoders: - - id: 0 - type: trt_decoder - block_size: 1 - syndrome_size: 1 - H_sparse: [0, -1] - O_sparse: [] - D_sparse: [0, -1] - decoder_custom_args: - onnx_load_path: /tmp/predecoder.onnx - global_decoder_params: - merge_strategy: smallest_weight -)"; + cudaqx::heterogeneous_map map; + map.insert("onnx_load_path", std::string("/tmp/predecoder.onnx")); + cudaqx::heterogeneous_map gd_params; + gd_params.insert("merge_strategy", std::string("smallest_weight")); + map.insert("global_decoder_params", gd_params); EXPECT_THROW( - cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( - yaml_params_without_decoder), + cudaq::qec::decoding::config::trt_decoder_config::from_heterogeneous_map( + map), std::runtime_error); + + cudaq::qec::decoding::config::trt_decoder_config trt_config; + trt_config.onnx_load_path = "/tmp/predecoder.onnx"; + auto pymatching_params = cudaq::qec::decoding::config::pymatching_config(); + pymatching_params.merge_strategy = "smallest_weight"; + trt_config.global_decoder_params = pymatching_params; + EXPECT_THROW(trt_config.to_heterogeneous_map(), std::runtime_error); } TEST(DecoderYAMLTest, SlidingWindowDecoder) { @@ -521,25 +480,29 @@ TEST(DecoderYAMLTest, SlidingWindowDecoder) { config.syndrome_size = n_rows; // Sliding window config + config.decoder_custom_args = + cudaq::qec::decoding::config::sliding_window_config(); + auto &sw_config = + std::get( + config.decoder_custom_args); config.H_sparse = cudaq::qec::pcm_to_sparse_vec(pcm); config.O_sparse = cudaq::qec::pcm_to_sparse_vec(cudaqx::tensor({2, n_cols})); config.D_sparse = cudaq::qec::generate_timelike_sparse_detector_matrix( config.syndrome_size, 2, /*include_first_round=*/false); - cudaqx::heterogeneous_map sw_args; - sw_args.insert("window_size", std::size_t{1}); - sw_args.insert("step_size", std::size_t{1}); - sw_args.insert("num_syndromes_per_round", n_syndromes_per_round); - sw_args.insert("straddle_start_round", false); - sw_args.insert("straddle_end_round", true); - sw_args.insert("error_rate_vec", std::vector(config.block_size, 0.1)); + sw_config.window_size = 1; + sw_config.step_size = 1; + sw_config.num_syndromes_per_round = n_syndromes_per_round; + sw_config.straddle_start_round = false; + sw_config.straddle_end_round = true; + sw_config.error_rate_vec = + std::vector(config.block_size, 0.1); // Inner decoder config - sw_args.insert("inner_decoder_name", "multi_error_lut"); - cudaqx::heterogeneous_map inner_lut_args; - inner_lut_args.insert("lut_error_depth", 2); - sw_args.insert("inner_decoder_params", inner_lut_args); - config.decoder_custom_args = sw_args; + sw_config.inner_decoder_name = "multi_error_lut"; + sw_config.multi_error_lut_params = + cudaq::qec::decoding::config::multi_error_lut_config(); + sw_config.multi_error_lut_params->lut_error_depth = 2; multi_config.decoders.push_back(config); @@ -547,20 +510,105 @@ TEST(DecoderYAMLTest, SlidingWindowDecoder) { test_decoder_creation(multi_config); } +TEST(DecoderConfigMapTest, SRelayNvQldpcAndTrtRoundTrip) { + using namespace cudaq::qec::decoding::config; + + srelay_bp_config relay; + relay.pre_iter = 3; + relay.num_sets = 5; + relay.stopping_criterion = "NConv"; + relay.stop_nconv = 2; + + auto relay_map = relay.to_heterogeneous_map(); + auto relay_from_map = srelay_bp_config::from_heterogeneous_map(relay_map); + EXPECT_EQ(relay_from_map, relay); + + nv_qldpc_decoder_config nv; + nv.use_sparsity = true; + nv.error_rate = 0.02; + nv.error_rate_vec = std::vector{0.1, 0.2, 0.3}; + nv.max_iterations = 8; + nv.n_threads = 4; + nv.use_osd = true; + nv.osd_method = 1; + nv.osd_order = 2; + nv.bp_batch_size = 16; + nv.osd_batch_size = 8; + nv.iter_per_check = 3; + nv.clip_value = 9.5; + nv.bp_method = 2; + nv.scale_factor = 0.75; + nv.proc_float = "fp64"; + nv.gamma0 = 0.4; + nv.gamma_dist = std::vector{0.1, 0.2}; + nv.explicit_gammas = std::vector>{{0.1, 0.2}, {0.3, 0.4}}; + nv.srelay_config = relay; + nv.bp_seed = 13; + nv.composition = 1; + + auto nv_map = nv.to_heterogeneous_map(); + auto nv_from_map = nv_qldpc_decoder_config::from_heterogeneous_map(nv_map); + EXPECT_EQ(nv_from_map, nv); + + cudaqx::heterogeneous_map nested_relay_map; + nested_relay_map.insert("pre_iter", std::size_t{7}); + nested_relay_map.insert("num_sets", std::size_t{9}); + nested_relay_map.insert("stopping_criterion", std::string("RelErr")); + nested_relay_map.insert("stop_nconv", std::size_t{4}); + cudaqx::heterogeneous_map nv_with_nested_relay; + nv_with_nested_relay.insert("srelay_config", nested_relay_map); + auto nv_from_nested = + nv_qldpc_decoder_config::from_heterogeneous_map(nv_with_nested_relay); + ASSERT_TRUE(nv_from_nested.srelay_config.has_value()); + EXPECT_EQ(nv_from_nested.srelay_config->pre_iter, std::size_t{7}); + EXPECT_EQ(nv_from_nested.srelay_config->num_sets, std::size_t{9}); + + trt_decoder_config trt; + trt.onnx_load_path = "/tmp/model.onnx"; + trt.engine_save_path = "/tmp/model.engine"; + trt.precision = "noTF32"; + trt.memory_workspace = std::size_t{4096}; + auto trt_map = trt.to_heterogeneous_map(); + auto trt_from_map = trt_decoder_config::from_heterogeneous_map(trt_map); + EXPECT_EQ(trt_from_map, trt); +} + +TEST(DecoderConfigMapTest, DecoderCustomArgsCoversNvQldpcAndTrtVariants) { + using namespace cudaq::qec::decoding::config; + + decoder_config nv_decoder; + nv_decoder.type = "nv-qldpc-decoder"; + nv_qldpc_decoder_config nv_args; + nv_args.max_iterations = 11; + nv_args.error_rate_vec = std::vector{0.1, 0.1}; + nv_decoder.decoder_custom_args = nv_args; + auto nv_map = nv_decoder.decoder_custom_args_to_heterogeneous_map(); + EXPECT_TRUE(nv_map.contains("max_iterations")); + EXPECT_EQ(nv_map.get("max_iterations"), 11); + + decoder_config trt_decoder; + trt_decoder.type = "trt_decoder"; + trt_decoder_config trt_args; + trt_args.engine_load_path = "/tmp/model.engine"; + trt_args.precision = "tf32"; + trt_decoder.decoder_custom_args = trt_args; + auto trt_map = trt_decoder.decoder_custom_args_to_heterogeneous_map(); + EXPECT_TRUE(trt_map.contains("engine_load_path")); + EXPECT_EQ(trt_map.get("engine_load_path"), "/tmp/model.engine"); +} + TEST(DecoderYAMLTest, TrtDecoderConfigRoundTripWithoutInstantiation) { - if (!is_trt_decoder_schema_available()) - GTEST_SKIP() << "trt_decoder plugin (and its parameter schema) not built"; using namespace cudaq::qec::decoding::config; multi_decoder_config multi_config; decoder_config config = create_test_empty_decoder_config(0); config.type = "trt_decoder"; - cudaqx::heterogeneous_map trt_args; - trt_args.insert("engine_load_path", "/tmp/prebuilt.engine"); - trt_args.insert("engine_save_path", "/tmp/saved.engine"); - trt_args.insert("precision", "best"); - trt_args.insert("memory_workspace", std::size_t{1 << 20}); - config.decoder_custom_args = trt_args; + trt_decoder_config trt_config; + trt_config.engine_load_path = "/tmp/prebuilt.engine"; + trt_config.engine_save_path = "/tmp/saved.engine"; + trt_config.precision = "best"; + trt_config.memory_workspace = std::size_t{1 << 20}; + config.decoder_custom_args = trt_config; multi_config.decoders.push_back(config); test_decoder_yaml_roundtrip(multi_config); @@ -569,7 +617,7 @@ TEST(DecoderYAMLTest, TrtDecoderConfigRoundTripWithoutInstantiation) { TEST(DecoderYAMLTest, SlidingWindowInnerDecoderVariantRoundTrips) { using namespace cudaq::qec::decoding::config; - auto check_roundtrip = [](const cudaqx::heterogeneous_map &sw_args) { + auto check_roundtrip = [](sliding_window_config sw_config) { multi_decoder_config multi_config; decoder_config config = create_test_empty_decoder_config(0); config.type = "sliding_window"; @@ -581,26 +629,26 @@ TEST(DecoderYAMLTest, SlidingWindowInnerDecoderVariantRoundTrips) { config.O_sparse = cudaq::qec::pcm_to_sparse_vec(O); config.D_sparse = cudaq::qec::generate_timelike_sparse_detector_matrix( config.syndrome_size, 2, /*include_first_round=*/false); - config.decoder_custom_args = sw_args; + config.decoder_custom_args = sw_config; multi_config.decoders.push_back(config); test_decoder_yaml_roundtrip(multi_config); }; - cudaqx::heterogeneous_map single_lut_sw; - single_lut_sw.insert("window_size", std::size_t{1}); - single_lut_sw.insert("step_size", std::size_t{1}); - single_lut_sw.insert("num_syndromes_per_round", std::size_t{2}); - single_lut_sw.insert("num_boundary_syndromes", std::size_t{1}); - single_lut_sw.insert("error_rate_vec", std::vector(6, 0.1)); - single_lut_sw.insert("inner_decoder_name", "single_error_lut"); + sliding_window_config single_lut_sw; + single_lut_sw.window_size = std::size_t{1}; + single_lut_sw.step_size = std::size_t{1}; + single_lut_sw.num_syndromes_per_round = std::size_t{2}; + single_lut_sw.error_rate_vec = std::vector(6, 0.1); + single_lut_sw.inner_decoder_name = "single_error_lut"; + single_lut_sw.single_error_lut_params = single_error_lut_config(); check_roundtrip(single_lut_sw); - auto nv_sw = single_lut_sw; - nv_sw.insert("inner_decoder_name", "nv-qldpc-decoder"); - cudaqx::heterogeneous_map nv_inner; - nv_inner.insert("max_iterations", 5); - nv_inner.insert("error_rate_vec", std::vector(6, 0.1)); - nv_sw.insert("inner_decoder_params", nv_inner); + sliding_window_config nv_sw = single_lut_sw; + nv_sw.inner_decoder_name = "nv-qldpc-decoder"; + nv_sw.single_error_lut_params.reset(); + nv_sw.nv_qldpc_decoder_params = nv_qldpc_decoder_config(); + nv_sw.nv_qldpc_decoder_params->max_iterations = 5; + nv_sw.nv_qldpc_decoder_params->error_rate_vec = std::vector(6, 0.1); check_roundtrip(nv_sw); } @@ -618,70 +666,6 @@ TEST(DecoderConfigTest, ConfigureRejectsDuplicateAndNegativeIds) { EXPECT_EQ(configure_decoders(negative_id), 3); } -TEST(DecoderConfigTest, CreateRealtimeDecoderConfiguresRuntimeState) { - auto config = create_test_sample_realtime_decoder_config(7); - - auto decoder = cudaq::qec::decoding::host::create_realtime_decoder(config); - - ASSERT_NE(decoder, nullptr); - EXPECT_EQ(decoder->get_decoder_id(), 7u); - EXPECT_EQ(decoder->get_num_observables(), 2u); - EXPECT_EQ(decoder->get_num_msyn_per_decode(), 20u); -} - -TEST(DecoderConfigTest, CreateRealtimeDecoderRequiresDetectorMatrix) { - auto config = create_test_sample_realtime_decoder_config(0); - config.D_sparse.clear(); - - EXPECT_THROW(cudaq::qec::decoding::host::create_realtime_decoder(config), - std::runtime_error); -} - -TEST(DecoderConfigTest, CreateRealtimeDecoderRejectsUnrepresentableId) { - auto config = create_test_sample_realtime_decoder_config(0); - config.id = - static_cast(std::numeric_limits::max()) + 1; - - EXPECT_THROW(cudaq::qec::decoding::host::create_realtime_decoder(config), - std::invalid_argument); -} - -TEST(DecoderConfigTest, SessionRegistryUsesConfiguredRealtimeDecoder) { - cudaq::qec::decoding::config::multi_decoder_config config; - auto decoder_config = create_test_sample_realtime_decoder_config(0); - config.decoders.push_back(std::move(decoder_config)); - - cudaq::qec::decoding_server::SessionRegistry registry; - registry.load_from_config(config, "unit test"); - - const auto &decoder = registry.get(0).dec; - ASSERT_NE(decoder, nullptr); - EXPECT_EQ(decoder->get_decoder_id(), 0u); - EXPECT_EQ(decoder->get_num_observables(), 2u); - EXPECT_EQ(decoder->get_num_msyn_per_decode(), 20u); -} - -TEST(DecoderConfigTest, SessionRegistryRejectsMissingDetectorMatrix) { - cudaq::qec::decoding::config::multi_decoder_config config; - auto decoder_config = create_test_sample_realtime_decoder_config(0); - decoder_config.D_sparse.clear(); - config.decoders.push_back(std::move(decoder_config)); - - cudaq::qec::decoding_server::SessionRegistry registry; - EXPECT_THROW(registry.load_from_config(config, "unit test"), - std::runtime_error); -} - -TEST(DecoderConfigTest, SessionRegistryRejectsNegativeDecoderId) { - cudaq::qec::decoding::config::multi_decoder_config config; - auto decoder_config = create_test_sample_realtime_decoder_config(-1); - config.decoders.push_back(std::move(decoder_config)); - - cudaq::qec::decoding_server::SessionRegistry registry; - EXPECT_THROW(registry.load_from_config(config, "unit test"), - std::runtime_error); -} - TEST(DecoderConfigTest, ConfigureFromFileWithDebugLogging) { using namespace cudaq::qec::decoding::config; @@ -712,394 +696,6 @@ TEST(DecoderConfigTest, ConfigureFromMissingFileReturnsError) { EXPECT_EQ(configure_decoders_from_file(missing_path.c_str()), 1); } -TEST(DecoderSchemaTest, ThirdPartySchemaRegistrationEnablesCustomArgs) { - using namespace cudaq::qec::decoding::config; - - // A third-party decoder plugin registers a parameter schema (normally from - // a static initializer in its own shared library); the YAML layer then - // accepts and round-trips its decoder_custom_args with no framework - // changes. - register_decoder_schema({"third_party_demo_engine", - { - {"gain", param_kind::f64}, - }}); - register_decoder_schema( - {"third_party_demo_decoder", - { - {"strength", param_kind::f64}, - {"passes", param_kind::int32}, - {"mode", param_kind::string, /*required=*/true}, - {"weights", param_kind::f64_vec}, - {"engine", param_kind::string}, - {"engine_params", param_kind::discriminated, false, "", "engine", - /*materialize_empty=*/true}, - }}); - - const std::string yaml = R"( -decoders: - - id: 0 - type: third_party_demo_decoder - block_size: 2 - syndrome_size: 1 - H_sparse: [0, -1] - O_sparse: [0, -1] - D_sparse: [0, -1] - decoder_custom_args: - strength: 1.5 - passes: 3 - mode: fast - weights: [0.25, 0.75] - engine: third_party_demo_engine -)"; - auto config = multi_decoder_config::from_yaml_str(yaml); - const auto &args = config.decoders[0].decoder_custom_args.map(); - EXPECT_EQ(args.get("strength"), 1.5); - EXPECT_EQ(args.get("passes"), 3); - EXPECT_EQ(args.get("mode"), "fast"); - EXPECT_EQ(args.get>("weights"), - (std::vector{0.25, 0.75})); - // The discriminated engine_params section is materialized (empty) because - // "engine" names a registered schema and materialize_empty is set. - ASSERT_TRUE(args.contains("engine_params")); - EXPECT_TRUE(args.get("engine_params").empty()); - - const auto emitted = config.to_yaml_str(200); - auto round_tripped = multi_decoder_config::from_yaml_str(emitted); - EXPECT_EQ(round_tripped, config); - EXPECT_EQ(round_tripped.to_yaml_str(200), emitted); - - // Unknown keys are rejected against the schema. - const std::string misspelled = R"( -decoders: - - id: 0 - type: third_party_demo_decoder - block_size: 2 - syndrome_size: 1 - H_sparse: [0, -1] - O_sparse: [0, -1] - D_sparse: [0, -1] - decoder_custom_args: - strenght: 1.5 - mode: fast -)"; - EXPECT_THROW(multi_decoder_config::from_yaml_str(misspelled), - std::runtime_error); - - // Missing required keys are rejected when the section is present. - const std::string missing_required = R"( -decoders: - - id: 0 - type: third_party_demo_decoder - block_size: 2 - syndrome_size: 1 - H_sparse: [0, -1] - O_sparse: [0, -1] - D_sparse: [0, -1] - decoder_custom_args: - strength: 1.5 -)"; - EXPECT_THROW(multi_decoder_config::from_yaml_str(missing_required), - std::runtime_error); - - // A populated discriminated section round-trips, and one that names an - // unregistered schema is rejected. - const std::string with_engine_params = R"( -decoders: - - id: 0 - type: third_party_demo_decoder - block_size: 2 - syndrome_size: 1 - H_sparse: [0, -1] - O_sparse: [0, -1] - D_sparse: [0, -1] - decoder_custom_args: - mode: fast - engine: third_party_demo_engine - engine_params: - gain: 2.5 -)"; - auto engine_config = multi_decoder_config::from_yaml_str(with_engine_params); - const auto &engine_args = engine_config.decoders[0].decoder_custom_args.map(); - EXPECT_EQ(engine_args.get("engine_params") - .get("gain"), - 2.5); - auto engine_round_tripped = - multi_decoder_config::from_yaml_str(engine_config.to_yaml_str(200)); - EXPECT_EQ(engine_round_tripped, engine_config); - - const std::string unknown_engine = R"( -decoders: - - id: 0 - type: third_party_demo_decoder - block_size: 2 - syndrome_size: 1 - H_sparse: [0, -1] - O_sparse: [0, -1] - D_sparse: [0, -1] - decoder_custom_args: - mode: fast - engine: engine_without_schema - engine_params: {} -)"; - EXPECT_THROW(multi_decoder_config::from_yaml_str(unknown_engine), - std::runtime_error); -} - -TEST(DecoderSchemaTest, CustomArgsForUnregisteredTypeThrow) { - const std::string yaml = R"( -decoders: - - id: 0 - type: decoder_without_registered_schema - block_size: 1 - syndrome_size: 1 - H_sparse: [0, -1] - O_sparse: [0, -1] - D_sparse: [0, -1] - decoder_custom_args: - anything: 1 -)"; - EXPECT_THROW( - cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str(yaml), - std::runtime_error); -} - -TEST(DecoderSchemaTest, ExamplePluginRegistersSchema) { - // The in-tree example decoder plugin registers a (parameter-less) schema - // from its own shared library; its presence here proves the end-to-end - // plugin registration path works. - EXPECT_NE(cudaq::qec::decoding::config::find_decoder_schema( - "single_error_lut_example"), - nullptr); -} - -TEST(DecoderSchemaTest, ValidateCustomArgsChecksProgrammaticMaps) { - using namespace cudaq::qec::decoding::config; - - // Maps built programmatically (or from Python dicts) never pass through the - // YAML parser, so validate_custom_args applies the same schema checks - // explicitly. - register_decoder_schema({"third_party_demo_engine", - { - {"gain", param_kind::f64}, - }}); - register_decoder_schema( - {"third_party_demo_decoder", - { - {"strength", param_kind::f64}, - {"passes", param_kind::int32}, - {"mode", param_kind::string, /*required=*/true}, - {"weights", param_kind::f64_vec}, - {"engine", param_kind::string}, - {"engine_params", param_kind::discriminated, false, "", "engine", - /*materialize_empty=*/true}, - }}); - - decoder_config config; - config.type = "third_party_demo_decoder"; - cudaqx::heterogeneous_map args; - args.insert("strength", 1.5); - args.insert("mode", std::string("fast")); - config.decoder_custom_args = args; - EXPECT_NO_THROW(config.validate_custom_args()); - - // Unknown key. - args.insert("strenght", 1.5); - config.decoder_custom_args = args; - EXPECT_THROW(config.validate_custom_args(), std::runtime_error); - - // Missing required key. - cudaqx::heterogeneous_map missing_mode; - missing_mode.insert("strength", 1.5); - config.decoder_custom_args = missing_mode; - EXPECT_THROW(config.validate_custom_args(), std::runtime_error); - - // Nested discriminated sections are validated with the schema named by the - // discriminator. - cudaqx::heterogeneous_map engine_params; - engine_params.insert("gain", 2.5); - cudaqx::heterogeneous_map with_engine; - with_engine.insert("mode", std::string("fast")); - with_engine.insert("engine", std::string("third_party_demo_engine")); - with_engine.insert("engine_params", engine_params); - config.decoder_custom_args = with_engine; - EXPECT_NO_THROW(config.validate_custom_args()); - - engine_params.insert("gian", 2.5); - with_engine.insert("engine_params", engine_params); - config.decoder_custom_args = with_engine; - EXPECT_THROW(config.validate_custom_args(), std::runtime_error); - - // Unregistered decoder types reject non-empty args (and accept empty ones). - decoder_config unregistered; - unregistered.type = "decoder_without_registered_schema"; - EXPECT_NO_THROW(unregistered.validate_custom_args()); - cudaqx::heterogeneous_map anything; - anything.insert("anything", 1); - unregistered.decoder_custom_args = anything; - EXPECT_THROW(unregistered.validate_custom_args(), std::runtime_error); - - // multi_decoder_config validates every decoder. - multi_decoder_config multi; - multi.decoders.push_back(unregistered); - EXPECT_THROW(multi.validate_custom_args(), std::runtime_error); -} - -TEST(DecoderSchemaTest, ProgrammaticConfigsMaterializeSchemaDefaults) { - using namespace cudaq::qec::decoding::config; - - // Schema-declared defaults (materialize_empty discriminated sections, e.g. - // trt_decoder's global_decoder_params) must apply to programmatically - // built configs at the decoder-construction seam, not only on the YAML - // parse path. - register_decoder_schema({"third_party_demo_engine", - { - {"gain", param_kind::f64}, - }}); - register_decoder_schema( - {"third_party_demo_decoder", - { - {"mode", param_kind::string, /*required=*/true}, - {"engine", param_kind::string}, - {"engine_params", param_kind::discriminated, false, "", "engine", - /*materialize_empty=*/true}, - }}); - - decoder_config config; - config.type = "third_party_demo_decoder"; - cudaqx::heterogeneous_map args; - args.insert("mode", std::string("fast")); - args.insert("engine", std::string("third_party_demo_engine")); - config.decoder_custom_args = args; - - auto materialized = config.decoder_custom_args_to_heterogeneous_map(); - ASSERT_TRUE(materialized.contains("engine_params")); - EXPECT_TRUE( - materialized.get("engine_params").empty()); - // The stored args are untouched; only the constructor-facing view defaults. - EXPECT_FALSE(config.decoder_custom_args.map().contains("engine_params")); - - // A decoder type without a registered schema passes its args through. - decoder_config unregistered; - unregistered.type = "decoder_without_registered_schema"; - unregistered.decoder_custom_args = args; - EXPECT_TRUE(custom_args_maps_equal( - unregistered.decoder_custom_args_to_heterogeneous_map(), args)); -} - -TEST(DecoderSchemaTest, CustomArgsEqualityIsSignAware) { - using namespace cudaq::qec::decoding::config; - - // size_t(2^64-1) must not compare equal to int(-1) via wraparound. - cudaqx::heterogeneous_map a; - a.insert("seed", std::numeric_limits::max()); - cudaqx::heterogeneous_map b; - b.insert("seed", int(-1)); - EXPECT_FALSE(custom_args_maps_equal(a, b)); - - // Same-value cross-width comparisons still hold. - cudaqx::heterogeneous_map c; - c.insert("seed", std::size_t(7)); - cudaqx::heterogeneous_map d; - d.insert("seed", int(7)); - EXPECT_TRUE(custom_args_maps_equal(c, d)); - cudaqx::heterogeneous_map e; - e.insert("seed", int(-1)); - EXPECT_TRUE(custom_args_maps_equal(b, e)); -} - -TEST(DecoderSchemaTest, SlidingWindowValidateHookRejectsBadWindowing) { - using namespace cudaq::qec::decoding::config; - - // The sliding_window schema registers a validate hook for the cross-field - // constraints its per-key specs can't express; the hook runs both when YAML - // is parsed and from validate_custom_args. - const std::string yaml_template = R"( -decoders: - - id: 0 - type: sliding_window - block_size: 2 - syndrome_size: 2 - H_sparse: [0, -1, 1, -1] - O_sparse: [0, -1, 1, -1] - D_sparse: [0, -1, 1, -1] - decoder_custom_args: - window_size: WINDOW - step_size: STEP - error_rate_vec: [0.01, 0.01] - inner_decoder_name: single_error_lut -)"; - auto make_yaml = [&](const std::string &window, const std::string &step) { - std::string yaml = yaml_template; - yaml.replace(yaml.find("WINDOW"), 6, window); - yaml.replace(yaml.find("STEP"), 4, step); - return yaml; - }; - - EXPECT_NO_THROW(multi_decoder_config::from_yaml_str(make_yaml("4", "2"))); - // step_size > window_size - EXPECT_THROW(multi_decoder_config::from_yaml_str(make_yaml("2", "4")), - std::runtime_error); - // step_size == 0 - EXPECT_THROW(multi_decoder_config::from_yaml_str(make_yaml("2", "0")), - std::runtime_error); - - decoder_config config; - config.type = "sliding_window"; - cudaqx::heterogeneous_map args; - args.insert("window_size", std::size_t(2)); - args.insert("step_size", std::size_t(4)); - args.insert("error_rate_vec", std::vector{0.01, 0.01}); - args.insert("inner_decoder_name", std::string("single_error_lut")); - config.decoder_custom_args = args; - EXPECT_THROW(config.validate_custom_args(), std::runtime_error); - - args.insert("step_size", std::size_t(2)); - config.decoder_custom_args = args; - EXPECT_NO_THROW(config.validate_custom_args()); - - // num_boundary_syndromes must be <= num_syndromes_per_round (the boundary - // layers can be narrower than the interior, never wider). - args.insert("num_syndromes_per_round", std::size_t(2)); - args.insert("num_boundary_syndromes", std::size_t(3)); - config.decoder_custom_args = args; - EXPECT_THROW(config.validate_custom_args(), std::runtime_error); - - args.insert("num_boundary_syndromes", std::size_t(2)); - config.decoder_custom_args = args; - EXPECT_NO_THROW(config.validate_custom_args()); - - args.insert("error_rate_vec", std::vector{}); - config.decoder_custom_args = args; - EXPECT_THROW(config.validate_custom_args(), std::runtime_error); -} - -TEST(DecoderSchemaTest, JsonSchemaExportReflectsRegistry) { - using namespace cudaq::qec::decoding::config; - - // Structural spot checks; the python test suite parses the document and - // exercises it against real YAML configurations with the jsonschema - // package. - const std::string text = decoder_config_json_schema(); - EXPECT_NE(text.find("\"https://json-schema.org/draft/2020-12/schema\""), - std::string::npos); - EXPECT_NE(text.find("\"decoder_params\""), std::string::npos); - EXPECT_NE(text.find("\"decoder_config\""), std::string::npos); - EXPECT_NE(text.find("\"sparse_matrix\""), std::string::npos); - - // Every registered schema (built-in and plugin-registered alike) has a - // $defs entry, referenced from the per-type dispatch. - for (const auto &name : registered_decoder_schema_names()) { - EXPECT_NE(text.find("\"" + name + "\""), std::string::npos) << name; - EXPECT_NE(text.find("\"#/$defs/decoder_params/" + name + "\""), - std::string::npos) - << name; - } - - // Required keys and unknown-key rejection are carried over. - EXPECT_NE(text.find("\"error_rate_vec\""), std::string::npos); - EXPECT_NE(text.find("\"additionalProperties\": false"), std::string::npos); -} - TEST(DecoderConfigTest, SimulationHostPointerWrappersForwardToHostRuntime) { using namespace cudaq::qec::decoding::config; @@ -1147,95 +743,13 @@ TEST(DecoderYAMLTest, PrepareDecoderParamsSurfacesCudaDeviceId) { auto params2 = cudaq::qec::decoding::host::prepare_decoder_params(config2); EXPECT_FALSE(params2.contains("cuda_device_id")); - // trt type: still surfaced on the trt branch. prepare_decoder_params only - // manipulates the params map (no schema lookup, no filesystem), so empty - // custom args exercise the trt path without needing the trt plugin. + // trt type: still surfaced on the trt branch. auto config3 = create_test_empty_decoder_config(2); config3.type = "trt_decoder"; + config3.decoder_custom_args = + cudaq::qec::decoding::config::trt_decoder_config{}; config3.cuda_device_id = 1; auto params3 = cudaq::qec::decoding::host::prepare_decoder_params(config3); ASSERT_TRUE(params3.contains("cuda_device_id")); EXPECT_EQ(params3.get("cuda_device_id"), 1); } - -TEST(DecoderYAMLTest, ValidateCustomArgsChecksValueKinds) { - // A validated map is guaranteed to serialize: every value must be readable - // as its schema kind's canonical storage type, not just have a known key. - using cudaq::qec::decoding::config::decoder_config; - - decoder_config config; - config.type = "nv-qldpc-decoder"; - - cudaqx::heterogeneous_map args; - args.insert("clip_value", std::string("oops")); // f64 param - config.decoder_custom_args = args; - try { - config.validate_custom_args(); - FAIL() << "expected kind mismatch to be rejected"; - } catch (const std::runtime_error &e) { - EXPECT_NE(std::string(e.what()).find("clip_value"), std::string::npos); - EXPECT_NE(std::string(e.what()).find("float"), std::string::npos); - } - - // A std::size_t stored under an f64 param (the generic conversion used - // for dicts assigned before `type` is set) is equally unreadable at - // emission and must be rejected too. - cudaqx::heterogeneous_map generic; - generic.insert("clip_value", std::size_t{2}); - config.decoder_custom_args = generic; - EXPECT_THROW(config.validate_custom_args(), std::runtime_error); - - // Canonically-typed values pass. - cudaqx::heterogeneous_map good; - good.insert("clip_value", 2.0); - good.insert("max_iterations", 50); - config.decoder_custom_args = good; - EXPECT_NO_THROW(config.validate_custom_args()); -} - -TEST(DecoderYAMLTest, TrtFirstEmissionMaterializesGlobalDecoderParams) { - if (!is_trt_decoder_schema_available()) - GTEST_SKIP() << "trt_decoder plugin (and its parameter schema) not built"; - // A programmatic config with only global_decoder set serializes with the - // defaulted empty global_decoder_params on FIRST emission (as the old - // typed path did), so emitted YAML is stable across round trips. - auto config = create_test_empty_decoder_config(0); - config.type = "trt_decoder"; - cudaqx::heterogeneous_map args; - args.insert("global_decoder", std::string("pymatching")); - config.decoder_custom_args = args; - - cudaq::qec::decoding::config::multi_decoder_config multi_config; - multi_config.decoders.push_back(config); - const auto first = multi_config.to_yaml_str(200); - EXPECT_NE(first.find("global_decoder_params"), std::string::npos); - - auto round_tripped = - cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str(first); - EXPECT_EQ(round_tripped.to_yaml_str(200), first); -} - -TEST(DecoderYAMLTest, NonSchemaKeysDroppedFromDecoderParamsAndEmission) { - // A key outside the registered schema can never round-trip through YAML, - // so the constructor-facing map must not contain it either: local decoders - // and remote targets see the same configuration. - auto config = create_test_empty_decoder_config(0); - config.type = "multi_error_lut"; - cudaqx::heterogeneous_map args; - args.insert("lut_error_depth", 2); - args.insert("not_a_real_param", 42); - config.decoder_custom_args = args; - - auto params = config.decoder_custom_args_to_heterogeneous_map(); - EXPECT_TRUE(params.contains("lut_error_depth")); - EXPECT_FALSE(params.contains("not_a_real_param")); - - cudaq::qec::decoding::config::multi_decoder_config multi_config; - multi_config.decoders.push_back(config); - const auto yaml = multi_config.to_yaml_str(200); - EXPECT_NE(yaml.find("lut_error_depth"), std::string::npos); - EXPECT_EQ(yaml.find("not_a_real_param"), std::string::npos); - - // The stored args are untouched -- only the derived views are filtered. - EXPECT_TRUE(config.decoder_custom_args.map().contains("not_a_real_param")); -} diff --git a/libs/qec/unittests/test_decoding_server_core.cpp b/libs/qec/unittests/test_decoding_server_core.cpp deleted file mode 100644 index 9fc9c5514..000000000 --- a/libs/qec/unittests/test_decoding_server_core.cpp +++ /dev/null @@ -1,350 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2026 NVIDIA Corporation & Affiliates. * - * All rights reserved. * - * * - * This source code and the accompanying materials are made available under * - * the terms of the Apache License 2.0 which accompanies this distribution. * - *******************************************************************************/ - -#include "DecodingServer.h" -#include "DecodingSession.h" -#include "RoundAccumulator.h" -#include "RpcDispatcher.h" -#include "RpcWireFormat.h" -#include "../lib/hardware_guards.h" - -#include "cudaq/qec/decoder.h" -#include "cudaq/qec/sparse_binary_matrix.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - -using namespace cudaq::qec::decoding_server; - -class ControlledDecoder final : public cudaq::qec::decoder { -public: - ControlledDecoder() - : decoder(cudaq::qec::sparse_binary_matrix::from_csr( - /*num_rows=*/1, /*num_cols=*/1, /*row_ptrs=*/{0, 1}, - /*col_indices=*/{0})) { - set_O_sparse(std::vector>{{0}}); - // One detector is the parity of two incoming measurement bits, so a decode - // completes only after two one-bit enqueue calls. - set_D_sparse(std::vector>{{0, 1}}); - } - - cudaq::qec::decoder_result - decode(const std::vector &syndrome) override { - if (throw_on_decode) - throw std::runtime_error("controlled decoder failure"); - cudaq::qec::decoder_result result; - result.converged = converged; - result.result = {syndrome.at(0)}; - return result; - } - - bool converged = false; - bool throw_on_decode = false; -}; - -class CaptureTransceiver final : public ITransceiver { -public: - RxFrame recv() override { return {}; } - - void send(const PeerId &, const uint8_t *data, std::size_t len) override { - response.assign(data, data + len); - } - - void shutdown() override {} - - std::vector response; -}; - -std::pair, ControlledDecoder *> -make_session() { - auto decoder = std::make_unique(); - auto *raw_decoder = decoder.get(); - SyndromeMappingTable mappings{{0, {{}}}}; - return {DecodingSession::create(std::move(decoder), std::move(mappings)), - raw_decoder}; -} - -WorkItem make_enqueue(CaptureTransceiver &transport, uint64_t counter, - const std::vector &bits) { - WorkItem item{}; - item.function_id = kEnqueueSyndromesFunctionId; - item.request_id = static_cast(counter + 1); - item.response_transport = &transport; - item.frame_buf.resize(sizeof(RPCHeader) + sizeof(EnqueuePayload) + - bit_packed_bytes(bits.size())); - - auto *request = reinterpret_cast(item.frame_buf.data() + - sizeof(RPCHeader)); - request->decoder_id = 0; - request->counter = static_cast(counter); - request->syndrome_mapping_id = 0; - request->num_syndromes = static_cast(bits.size()); - - auto *packed = - item.frame_buf.data() + sizeof(RPCHeader) + sizeof(EnqueuePayload); - for (std::size_t i = 0; i < bits.size(); ++i) - if (bits[i] & 1u) - packed[i / 8] |= static_cast(1u << (i % 8)); - return item; -} - -WorkItem make_get_corrections(CaptureTransceiver &transport, bool reset) { - WorkItem item{}; - item.function_id = kGetCorrectionsFunctionId; - item.request_id = 101; - item.response_transport = &transport; - item.frame_buf.resize(sizeof(RPCHeader) + sizeof(GetCorrectionsPayload)); - - auto *request = reinterpret_cast( - item.frame_buf.data() + sizeof(RPCHeader)); - request->decoder_id = 0; - request->return_size = 1; - request->reset = reset ? 1 : 0; - return item; -} - -WorkItem make_reset(CaptureTransceiver &transport) { - WorkItem item{}; - item.function_id = kResetDecoderFunctionId; - item.request_id = 202; - item.response_transport = &transport; - item.frame_buf.resize(sizeof(RPCHeader) + sizeof(ResetPayload)); - auto *request = reinterpret_cast(item.frame_buf.data() + - sizeof(RPCHeader)); - request->decoder_id = 0; - return item; -} - -void expect_status(const CaptureTransceiver &transport, RpcStatus status) { - ASSERT_GE(transport.response.size(), sizeof(RPCResponse)); - const auto *response = - reinterpret_cast(transport.response.data()); - EXPECT_EQ(response->magic, kRPCResponseMagic); - EXPECT_EQ(response->status, static_cast(status)); -} - -TEST(DecodingSessionStateTest, RequiresACompletedDecodeForEachResult) { - auto [session, decoder] = make_session(); - CaptureTransceiver transport; - - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::NOT_READY); - - session->on_enqueue(make_enqueue(transport, 0, {1})); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::NOT_READY); - - // A completed decode is ready even when the algorithm reports that it did - // not converge. Readiness and convergence are different contracts. - ASSERT_FALSE(decoder->converged); - session->on_enqueue(make_enqueue(transport, 1, {0})); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::OK); - ASSERT_EQ(transport.response.size(), sizeof(RPCResponse) + 1); - EXPECT_EQ(transport.response[sizeof(RPCResponse)] & 1u, 1u); - - // Accepting part of the next volume makes the previous result stale. - session->on_enqueue(make_enqueue(transport, 2, {0})); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::NOT_READY); - - session->on_enqueue(make_enqueue(transport, 3, {0})); - session->on_get_corrections(make_get_corrections(transport, true)); - expect_status(transport, RpcStatus::OK); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::NOT_READY); -} - -TEST(DecodingSessionStateTest, KeepsFailuresStickyUntilReset) { - auto [session, decoder] = make_session(); - CaptureTransceiver transport; - - decoder->throw_on_decode = true; - session->on_enqueue(make_enqueue(transport, 0, {1})); - session->on_enqueue(make_enqueue(transport, 1, {0})); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::INTERNAL_ERROR); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::INTERNAL_ERROR); - - decoder->throw_on_decode = false; - session->on_reset(make_reset(transport)); - expect_status(transport, RpcStatus::OK); - session->on_enqueue(make_enqueue(transport, 2, {1})); - session->on_enqueue(make_enqueue(transport, 3, {0})); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::OK); - - session->latch_syndromes_dropped(); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::SYNDROMES_DROPPED); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::SYNDROMES_DROPPED); - - session->on_reset(make_reset(transport)); - expect_status(transport, RpcStatus::OK); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::NOT_READY); -} - -TEST(DecodingSessionStateTest, RejectsMeasurementVolumeOverflow) { - auto [session, decoder] = make_session(); - CaptureTransceiver transport; - (void)decoder; - - session->on_enqueue(make_enqueue(transport, 0, {1})); - session->on_enqueue(make_enqueue(transport, 1, {0, 1})); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::INTERNAL_ERROR); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::INTERNAL_ERROR); - - session->on_reset(make_reset(transport)); - expect_status(transport, RpcStatus::OK); - session->on_get_corrections(make_get_corrections(transport, false)); - expect_status(transport, RpcStatus::NOT_READY); -} - -TEST(RoundAccumulatorTest, RejectsMultiVpPassThroughMappings) { - const RoundKey key{.decoder_id = 0, .counter = 12, .syndrome_mapping_id = 0}; - const SyndromeMappingTable multi_vp{{0, {{}, {}}}}; - - RoundAccumulator unequal_lengths; - const uint8_t vp0[] = {1}; - EXPECT_THROW(unequal_lengths.ingest(key, 0, vp0, 1, multi_vp), - std::invalid_argument); - - RoundAccumulator equal_lengths; - const uint8_t vp0_equal[] = {1, 0}; - EXPECT_THROW(equal_lengths.ingest(key, 0, vp0_equal, 2, multi_vp), - std::invalid_argument); - - RoundAccumulator single_vp; - const SyndromeMappingTable supported{{0, {{}}}}; - auto completed = single_vp.ingest(key, 0, vp0_equal, 2, supported); - ASSERT_TRUE(completed.has_value()); - EXPECT_EQ(completed->bits, (std::vector{1, 0})); -} - -TEST(RpcDispatcherTest, ConvertsHandlerExceptionsToErrorResponses) { - constexpr uint32_t function_id = 0x12345678; - RpcDispatcher dispatcher; - dispatcher.register_handler(function_id, - [](RxFrame, ResponseWriter &) -> void { - throw std::runtime_error("handler failure"); - }); - - RxFrame frame; - frame.buf.resize(sizeof(RPCHeader)); - auto *header = reinterpret_cast(frame.buf.data()); - header->magic = kRPCRequestMagic; - header->function_id = function_id; - header->request_id = 55; - - CaptureTransceiver transport; - EXPECT_NO_THROW(dispatcher.dispatch(std::move(frame), transport)); - expect_status(transport, RpcStatus::INTERNAL_ERROR); -} - -TEST(ResolveDecodeDevice, UnpinnedDefaultsToZero) { - EXPECT_EQ(cudaq::qec::decoding_server::resolve_decode_device(-1), 0); -} - -TEST(ResolveDecodeDevice, PinSelectsDevice) { - EXPECT_EQ(cudaq::qec::decoding_server::resolve_decode_device(3), 3); -} - -TEST(SetCudaDeviceForDecode, UnpinnedIsNoOp) { - // -1 = unpinned: must never touch the device or throw, even on a machine - // with no CUDA devices at all. - EXPECT_NO_THROW(cudaq::qec::detail_affinity::set_cuda_device_for_decode(-1)); -} - -TEST(SetCudaDeviceForDecode, ImpossibleDeviceThrows) { - // The handshake's failure transport rides on this throw; an id beyond the - // device count fails cudaSetDevice on any machine, including GPU-less CI. - int count = 0; - if (cudaGetDeviceCount(&count) != cudaSuccess) - count = 0; - EXPECT_THROW( - cudaq::qec::detail_affinity::set_cuda_device_for_decode(count + 7), - std::runtime_error); -} - -/// cuda_device_id_ is protected: setting an impossible id directly bypasses -/// decoder::get()'s construction-time range check, the only front door -- -/// which is exactly what makes the handshake's failure path injectable here. -class MispinnedDecoder final : public cudaq::qec::decoder { -public: - MispinnedDecoder() - : decoder(cudaq::qec::sparse_binary_matrix::from_csr( - /*num_rows=*/1, /*num_cols=*/1, /*row_ptrs=*/{0, 1}, - /*col_indices=*/{0})) { - set_O_sparse(std::vector>{{0}}); - set_D_sparse(std::vector>{{0, 1}}); - cuda_device_id_ = 1 << 20; - } - cudaq::qec::decoder_result - decode(const std::vector &) override { - return {}; - } -}; - -TEST(DecodingSessionPinHandshake, UnhonorablePinFailsStartWorker) { - // The contract under test: a worker that cannot pin must never serve, and - // the failure must surface on the caller (server-startup) thread. This is - // the test that fails if start_worker ever reverts to log-and-continue. - SyndromeMappingTable table; - table[0] = {{}}; - auto session = DecodingSession::create(std::make_unique(), - std::move(table)); - EXPECT_THROW(session->start_worker(), std::runtime_error); - // The failed worker was joined inside start_worker; nothing is left to - // serve and destruction must not hang. - EXPECT_FALSE(session->worker.joinable()); -} - -TEST(DecodingSessionPinHandshake, PinnedWorkerStartsAndServes) { - // start_worker() must resolve the pin handshake (throwing on failure per - // its contract) and leave a live worker serving items. - int count = 0; - if (cudaGetDeviceCount(&count) != cudaSuccess || count < 1) - GTEST_SKIP() << "needs >= 1 CUDA device"; - - cudaqx::heterogeneous_map params; - params.insert("cuda_device_id", 0); - auto dec = cudaq::qec::decoder::get( - "single_error_lut", - cudaq::qec::sparse_binary_matrix::from_csr(1, 1, {0, 1}, {0}), params); - dec->set_O_sparse(std::vector>{{0}}); - dec->set_D_sparse(std::vector>{{0, 1}}); - - SyndromeMappingTable table; - table[0] = {{}}; - auto session = DecodingSession::create(std::move(dec), std::move(table)); - ASSERT_NO_THROW(session->start_worker()); - - CaptureTransceiver transport; - ASSERT_TRUE(session->try_enqueue(make_reset(transport))); - for (int i = 0; i < 200 && session->reset_count.load() == 0; ++i) - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - EXPECT_EQ(session->reset_count.load(), 1u) - << "pinned worker did not serve the queued item"; -} - -} // namespace diff --git a/libs/qec/unittests/utils/CMakeLists.txt b/libs/qec/unittests/utils/CMakeLists.txt index 29eca5166..908ad1a81 100644 --- a/libs/qec/unittests/utils/CMakeLists.txt +++ b/libs/qec/unittests/utils/CMakeLists.txt @@ -171,9 +171,7 @@ add_executable(hololink_fpga_syndrome_playback hololink_fpga_syndrome_playback.cpp) target_include_directories(hololink_fpga_syndrome_playback - PRIVATE "${HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR}/src" - # cudaq/qec/realtime/decoder_rpc_ids.h (per-round RPC wire format). - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../include") + PRIVATE "${HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR}/src") if (DEFINED CUDAQ_INCLUDE_DIR AND CUDAQ_INCLUDE_DIR) target_include_directories(hololink_fpga_syndrome_playback @@ -260,11 +258,6 @@ if (GPU_ROCE_TRANSCEIVER_LIB AND CUDAQ_REALTIME_INCLUDE_DIR AND ${HOLOLINK_COMMON_LIB} ${CUDAQ_REALTIME_LIBRARY} cudaq-qec-realtime-decoding - # Device-graph scheduler path: absorb the proprietary cudevice archive - # WHOLE so the enqueue/get/reset DEVICE_CALL handlers + register/populate - # shims merge into this exe's cubin (device-linked with the dispatch - # kernel) and stay in the dynamic table for the plugin's dlsym lookups. - $<$:$> $<$:${_CUDAQ_LIBRARY}> $<$:${_NVQIR_LIBRARY}> CUDA::cudart @@ -279,10 +272,6 @@ if (GPU_ROCE_TRANSCEIVER_LIB AND CUDAQ_REALTIME_INCLUDE_DIR AND target_link_options(hololink_qldpc_graph_decoder_bridge PRIVATE "LINKER:--allow-shlib-undefined" - # Export the proprietary register/populate C-ABI shims so the dlopen'd - # nv-qldpc plugin (capture_decode_graph) and this bridge resolve them via - # dlsym(RTLD_DEFAULT, ...). - "LINKER:--export-dynamic" ) if (holoscan_FOUND) diff --git a/libs/qec/unittests/utils/hololink_fpga_syndrome_playback.cpp b/libs/qec/unittests/utils/hololink_fpga_syndrome_playback.cpp index 8ac31a1d3..37ba3b040 100644 --- a/libs/qec/unittests/utils/hololink_fpga_syndrome_playback.cpp +++ b/libs/qec/unittests/utils/hololink_fpga_syndrome_playback.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -29,7 +28,6 @@ #include #include -#include "cudaq/qec/realtime/decoder_rpc_ids.h" #include "cudaq/realtime/daemon/dispatcher/dispatch_kernel_launch.h" namespace { @@ -142,35 +140,6 @@ std::uint64_t parse_scalar(const std::string &content, } } -/// @brief Derive num_observables from O_sparse in the config. -/// -/// O_sparse encodes each observable as a row of correction indices terminated -/// by -1, so the number of row terminators is the number of observables. -std::size_t derive_num_observables(const std::string &content) { - std::size_t pos = content.find("O_sparse:"); - if (pos == std::string::npos) - return 0; - std::size_t bracket_start = content.find('[', pos); - if (bracket_start == std::string::npos) - return 0; - std::size_t bracket_end = content.find(']', bracket_start); - if (bracket_end == std::string::npos) - return 0; - std::string arr = - content.substr(bracket_start + 1, bracket_end - bracket_start - 1); - - std::size_t rows = 0; - std::istringstream ss(arr); - std::string token; - while (std::getline(ss, token, ',')) { - token.erase(0, token.find_first_not_of(" \t\n\r")); - token.erase(token.find_last_not_of(" \t\n\r") + 1); - if (token == "-1") - ++rows; - } - return rows; -} - /// @brief Derive num_measurements from D_sparse in the config. /// /// D_sparse encodes the detector-measurement matrix in row-major order with @@ -212,11 +181,7 @@ std::size_t derive_num_measurements(const std::string &content) { // ============================================================================ struct SyndromeEntry { - std::vector measurements; ///< flat concat of all rounds - /// Per-round measurement slices (from ROUND_START markers). Used by the - /// --per-round playback mode to emit one enqueue frame per round. Empty (or - /// a single slice) when the syndrome file has no per-round structure. - std::vector> per_round; + std::vector measurements; std::uint8_t expected_correction; }; @@ -228,41 +193,27 @@ std::vector load_syndromes(const std::string &path, return entries; std::string line; - std::vector current_shot; // flat (all rounds) - std::vector current_round; // bits since ROUND_START - std::vector> current_rounds; // rounds of this shot + std::vector current_shot; std::vector> shots; - std::vector>> shots_rounds; std::vector corrections; bool reading_shot = false; bool reading_corrections = false; - bool saw_round_marker = false; - - auto close_round = [&]() { - if (!current_round.empty()) { - current_rounds.push_back(current_round); - current_round.clear(); - } - }; - auto close_shot = [&]() { - close_round(); - if (reading_shot && !current_shot.empty()) { - shots.push_back(current_shot); - shots_rounds.push_back(current_rounds); - } - current_shot.clear(); - current_rounds.clear(); - }; while (std::getline(file, line)) { if (line.find("SHOT_START") == 0) { - close_shot(); + if (reading_shot && !current_shot.empty()) { + shots.push_back(current_shot); + } + current_shot.clear(); reading_shot = true; reading_corrections = false; continue; } if (line == "CORRECTIONS_START") { - close_shot(); + if (reading_shot && !current_shot.empty()) { + shots.push_back(current_shot); + } + current_shot.clear(); reading_shot = false; reading_corrections = true; continue; @@ -272,11 +223,6 @@ std::vector load_syndromes(const std::string &path, } if (line.find("NUM_DATA") == 0 || line.find("NUM_LOGICAL") == 0) { continue; - } else if (line.find("ROUND_START") == 0) { - // A new round within the current shot: close the prior round slice. - saw_round_marker = true; - close_round(); - continue; } else if (reading_shot) { line.erase(0, line.find_first_not_of(" \t\n\r")); line.erase(line.find_last_not_of(" \t\n\r") + 1); @@ -285,7 +231,6 @@ std::vector load_syndromes(const std::string &path, try { int bit = std::stoi(line); current_shot.push_back(static_cast(bit)); - current_round.push_back(static_cast(bit)); } catch (...) { } } else if (reading_corrections) { @@ -300,8 +245,10 @@ std::vector load_syndromes(const std::string &path, } } } - close_shot(); - (void)saw_round_marker; + + if (reading_shot && !current_shot.empty()) { + shots.push_back(current_shot); + } for (std::size_t i = 0; i < shots.size(); ++i) { if (shots[i].size() < syndrome_size) { @@ -311,12 +258,6 @@ std::vector load_syndromes(const std::string &path, } SyndromeEntry entry{}; entry.measurements = std::move(shots[i]); - entry.per_round = (i < shots_rounds.size()) - ? std::move(shots_rounds[i]) - : std::vector>{}; - // No per-round markers: treat the whole shot as a single round. - if (entry.per_round.empty()) - entry.per_round.push_back(entry.measurements); entry.expected_correction = (i < corrections.size()) ? corrections[i] : 0; entries.push_back(std::move(entry)); } @@ -347,70 +288,6 @@ build_rpc_payload(const std::vector &measurements, return payload; } -/// Build a per-round `enqueue_syndromes` RPC frame for the device-graph -/// scheduler (decoder_server_runtime.md#enqueue_syndromes): -/// [RPCHeader (fid=enqueue, arg_len)][EnqueueRequestPayload (32B)] -/// [ceil(n/8) LSB-first bit-packed syndrome bytes, no pad]. -/// One input byte (0/1) per syndrome bit. ptp_timestamp left 0 (FPGA fills -/// it). -std::vector -build_enqueue_frame(const std::vector &round_bits, - std::uint32_t request_id, std::int64_t decoder_id, - std::int64_t counter) { - namespace rpc = cudaq::qec::decoding::rpc; - const std::uint64_t n = round_bits.size(); - const std::size_t packed = rpc::bit_packed_bytes(n); - const std::size_t arg_len = sizeof(rpc::EnqueueRequestPayload) + packed; - std::vector payload( - sizeof(cudaq::realtime::RPCHeader) + arg_len, 0); - auto *header = reinterpret_cast(payload.data()); - header->magic = cudaq::realtime::RPC_MAGIC_REQUEST; - header->function_id = rpc::kEnqueueSyndromesFunctionId; - header->arg_len = static_cast(arg_len); - header->request_id = request_id; - header->ptp_timestamp = 0; - - auto *body = reinterpret_cast( - payload.data() + sizeof(cudaq::realtime::RPCHeader)); - body->decoder_id = decoder_id; - body->counter = counter; - body->syndrome_mapping_id = 0; - body->num_syndromes = static_cast(n); - - std::uint8_t *bits = payload.data() + sizeof(cudaq::realtime::RPCHeader) + - sizeof(rpc::EnqueueRequestPayload); - for (std::uint64_t i = 0; i < n; ++i) - if (round_bits[i] & 0x1u) - bits[i >> 3] |= static_cast(1u << (i & 7)); - return payload; -} - -/// Build a `get_corrections` RPC frame (decoder_server_runtime.md): -/// [RPCHeader (fid=get_corrections, arg_len=17)] -/// [GetCorrectionsRequestPayload{decoder_id, return_size, reset}]. -std::vector build_get_corrections_frame(std::uint32_t request_id, - std::int64_t decoder_id, - std::int64_t return_size, - std::uint8_t reset) { - namespace rpc = cudaq::qec::decoding::rpc; - const std::size_t arg_len = sizeof(rpc::GetCorrectionsRequestPayload); - std::vector payload( - sizeof(cudaq::realtime::RPCHeader) + arg_len, 0); - auto *header = reinterpret_cast(payload.data()); - header->magic = cudaq::realtime::RPC_MAGIC_REQUEST; - header->function_id = rpc::kGetCorrectionsFunctionId; - header->arg_len = static_cast(arg_len); - header->request_id = request_id; - header->ptp_timestamp = 0; - - auto *body = reinterpret_cast( - payload.data() + sizeof(cudaq::realtime::RPCHeader)); - body->decoder_id = decoder_id; - body->return_size = return_size; - body->reset = reset; - return payload; -} - // ============================================================================ // Command-Line Options // ============================================================================ @@ -426,11 +303,6 @@ struct Options { std::optional num_shots; bool verify = false; - // Per-round mode (device-graph scheduler): emit N enqueue_syndromes frames - // (one per round) followed by 1 get_corrections frame per shot, instead of a - // single full-window request. request_id = shot*(rounds+1)+local_frame. - bool per_round = false; - // When set, bypass BOOTP enumeration and connect directly to this UDP port. // Used with the FPGA emulator which doesn't support BOOTP. std::optional control_port; @@ -469,16 +341,7 @@ void print_usage(const char *argv0) { << " --spacing Inter-shot spacing in microseconds " "(default: 10)\n" << " --verify Capture and verify correction responses " - "via ILA\n" - << " --per-round Per-round protocol (device-graph scheduler): " - "send\n" - << " N enqueue_syndromes frames (one per " - "ROUND_START\n" - << " slice) + 1 get_corrections frame per shot. " - "Note:\n" - << " multiplies frames by rounds+1, so the 512-" - "cycle\n" - << " BRAM limits --num-shots accordingly.\n\n" + "via ILA\n\n" << "Emulator mode (bypass BOOTP enumeration):\n" << " --control-port UDP control port of the emulator\n" << " --vp-address VP register base (default: 0x1000)\n" @@ -517,8 +380,6 @@ Options parse_args(int argc, char **argv) { static_cast(std::stoul(argv[++i], nullptr, 0)); } else if (arg == "--verify") { options.verify = true; - } else if (arg == "--per-round") { - options.per_round = true; } else if (arg == "--qp-number" && i + 1 < argc) { options.qp_number = static_cast(std::stoul(argv[++i], nullptr, 0)); @@ -858,10 +719,7 @@ int64_t ptp_delta_ns(PtpTimestamp send, PtpTimestamp recv) { } struct LatencySample { - uint32_t request_id; ///< echoed request_id of the frame - uint32_t shot; ///< request_id / frames_per_shot (per-round) - uint32_t local; ///< frame index within the shot (per-round); else 0 - bool is_corr; ///< true = get_corrections frame, false = enqueue ACK + uint32_t msg_id; uint32_t send_sec, send_nsec; uint32_t recv_sec, recv_nsec; int64_t delta_ns; @@ -877,10 +735,10 @@ struct VerifyResult { std::size_t responses_matched = 0; std::size_t header_errors = 0; std::size_t correction_errors = 0; + std::size_t rpc_requests = 0; std::size_t rpc_responses = 0; std::size_t non_rpc_frames = 0; std::size_t tvalid_zero = 0; - std::size_t enqueue_acks = 0; ///< per-round: result_len==0 ACK responses std::vector latency_samples; }; @@ -896,17 +754,10 @@ struct VerifyResult { /// byte [24] correction value VerifyResult verify_captured_responses( const std::vector> &samples, - const std::vector &syndromes, std::size_t num_expected, - bool per_round = false, std::size_t frames_per_shot = 1, - std::size_t ring_depth = 64) { + const std::vector &syndromes, std::size_t num_expected) { VerifyResult result; result.total_samples = samples.size(); std::set shots_seen; - // DIAGNOSTIC census: count how many RPC responses carry each request_id, so a - // duplicate-processed frame (count==2) and a dropped frame (count==0) are - // identified exactly -- and we can check whether they alias the same ring - // slot (rid % num_pages). - std::map rid_count; for (std::size_t i = 0; i < samples.size(); ++i) { const auto &sample = samples[i]; @@ -927,94 +778,52 @@ VerifyResult verify_captured_responses( std::uint8_t correction_byte = data_bytes[sizeof(cudaq::realtime::RPCResponse)]; - // The ILA captures the FPGA-bound (TX) response path only, so every RPC - // frame here carries RPC_MAGIC_RESPONSE; anything else is a non-RPC frame - // (e.g. a continuation beat of a multi-beat AXI transfer). + if (resp.magic == cudaq::realtime::RPC_MAGIC_REQUEST) { + result.rpc_requests++; + continue; + } if (resp.magic != cudaq::realtime::RPC_MAGIC_RESPONSE) { result.non_rpc_frames++; continue; } result.rpc_responses++; - if (per_round) - rid_count[resp.request_id]++; if (resp.status != 0) { result.header_errors++; continue; } - // Frame identity (per-round): shot, local frame index within the shot, and - // whether this is the get_corrections frame (result_len>0) vs an enqueue - // ACK (result_len==0). - const std::size_t frames = frames_per_shot ? frames_per_shot : 1; - const std::uint32_t shot_index = - per_round ? static_cast(resp.request_id / frames) - : resp.request_id; - const std::uint32_t local = - per_round ? static_cast(resp.request_id % frames) : 0; - const bool is_corr = (!per_round) || (resp.result_len != 0); - - // PTP round-trip latency for EVERY response -- enqueue ACKs AND - // get_corrections. The FPGA injects a send timestamp into every playback - // frame and the ILA captures a recv timestamp for every captured sample, - // so all frames carry valid PTP. Recorded here (before the ACK skip - // below) so the CSV has one row per captured frame, not just one per shot. - { - uint64_t send_raw = extract_echoed_ptp_timestamp(resp); - uint64_t recv_raw = extract_ila_ptp_timestamp(sample); - if (send_raw != 0 && recv_raw != 0) { - auto send_ts = decode_ptp(send_raw); - auto recv_ts = decode_ptp(recv_raw); - int64_t delta = ptp_delta_ns(send_ts, recv_ts); - result.latency_samples.push_back({resp.request_id, shot_index, local, - is_corr, send_ts.sec, send_ts.nsec, - recv_ts.sec, recv_ts.nsec, delta}); - } - } - - // Per-round: enqueue_syndromes responses are empty ACKs (result_len==0); - // only get_corrections responses (result_len>0) carry corrections. - if (per_round && resp.result_len == 0) { - result.enqueue_acks++; - // DIAGNOSTIC: an empty response at the get_corrections frame position - // (local index == rounds) would be a get_corrections that came back - // result_len==0 (the old 99/100 anomaly). Dump what we know about it. - const std::size_t rounds = frames ? frames - 1 : 0; - if (local == rounds) { - std::cout << " [ANOMALY] sample " << i << ": result_len==0 at " - << "get_corrections position; request_id=" << resp.request_id - << " shot=" << shot_index << " status=" << resp.status - << " magic=0x" << std::hex << resp.magic << std::dec - << " corr_byte=" << static_cast(correction_byte) << "\n"; - } - continue; - } - + std::uint32_t shot_index = resp.request_id; if (shot_index >= syndromes.size()) { - std::cout << " Sample " << i << ": request_id=" << resp.request_id - << " -> shot=" << shot_index + std::cout << " Sample " << i << ": request_id=" << shot_index << " out of range (num_shots=" << syndromes.size() << ") [WARN]\n"; result.correction_errors++; continue; } - // Per-round get_corrections returns bit-packed observables (LSB-first); - // compare observable bit 0 to the expected logical correction. The - // shot-based path compares the raw correction byte as before. - std::uint8_t got = per_round - ? static_cast(correction_byte & 0x1u) - : correction_byte; std::uint8_t expected = syndromes[shot_index].expected_correction; - if (got == expected) { + if (correction_byte == expected) { result.responses_matched++; } else { - std::cout << " Sample " << i << " request_id=" << resp.request_id - << " shot=" << shot_index << ": got=" << static_cast(got) + std::cout << " Sample " << i << " request_id=" << shot_index + << ": got=" << static_cast(correction_byte) << " expected=" << static_cast(expected) << " [FAIL]\n"; result.correction_errors++; } + // PTP round-trip latency: send timestamp from response header, + // receive timestamp from ILA bits [584:521]. + uint64_t send_raw = extract_echoed_ptp_timestamp(resp); + uint64_t recv_raw = extract_ila_ptp_timestamp(sample); + if (send_raw != 0 && recv_raw != 0) { + auto send_ts = decode_ptp(send_raw); + auto recv_ts = decode_ptp(recv_raw); + int64_t delta = ptp_delta_ns(send_ts, recv_ts); + result.latency_samples.push_back({shot_index, send_ts.sec, send_ts.nsec, + recv_ts.sec, recv_ts.nsec, delta}); + } + shots_seen.insert(shot_index); } @@ -1022,46 +831,6 @@ VerifyResult verify_captured_responses( std::cout << " Unique shots verified: " << shots_seen.size() << " of " << num_expected << "\n"; - // DIAGNOSTIC: list any shots that never produced a get_corrections response. - if (shots_seen.size() < num_expected) { - std::cout << " [ANOMALY] missing shots (no correction response):"; - for (std::uint32_t s = 0; s < num_expected; ++s) - if (!shots_seen.count(s)) - std::cout << " " << s; - std::cout << "\n"; - } - - // DIAGNOSTIC: per-request_id census -- report duplicated (count>1) and - // dropped (count==0) frames with their (shot, local-frame) and ring slot, to - // confirm a slot-aliasing race between the scheduler's flag clear and the - // Hololink RX kernel refilling reused slots. - if (per_round && frames_per_shot) { - const std::uint32_t total_frames = - static_cast(num_expected * frames_per_shot); - auto describe = [&](std::uint32_t rid) { - const std::uint32_t local = rid % frames_per_shot; - const std::uint32_t shot = rid / frames_per_shot; - const char *kind = - (local + 1 == frames_per_shot) ? "get_corrections" : "enqueue"; - std::cout << " rid=" << rid << " (shot=" << shot << " local=" << local - << " " << kind << " slot%" << ring_depth << "=" - << (ring_depth ? rid % ring_depth : 0) << ")"; - }; - bool any = false; - for (std::uint32_t rid = 0; rid < total_frames; ++rid) { - int c = rid_count.count(rid) ? rid_count[rid] : 0; - if (c != 1) { - if (!any) { - std::cout << " [ANOMALY] request_id census (expected each seen " - "exactly once):\n"; - any = true; - } - describe(rid); - std::cout << " seen " << c << " times\n"; - } - } - } - return result; } @@ -1115,18 +884,6 @@ int main(int argc, char **argv) { if (num_measurements == 0) num_measurements = syndrome_size; - const std::size_t num_observables = derive_num_observables(config_content); - if (options.per_round && num_observables == 0) { - std::cerr << "Per-round mode requires O_sparse in config file\n"; - return 1; - } - if (options.per_round && num_observables != 1) { - std::cerr << "Per-round playback verification currently supports exactly " - "one observable (found " - << num_observables << " from O_sparse)\n"; - return 1; - } - auto syndromes = load_syndromes(syndromes_path, num_measurements); if (syndromes.empty()) { std::cerr << "No syndrome data loaded from " << syndromes_path << "\n"; @@ -1153,95 +910,32 @@ int main(int argc, char **argv) { : options.function_name) << " (id=0x" << std::hex << function_id << std::dec << ")\n"; - // Per-round shape (device-graph scheduler protocol): each shot becomes - // `rounds` enqueue_syndromes frames + 1 get_corrections frame. rounds is the - // number of ROUND_START slices in the syndrome file (1 if unstructured). - const std::size_t rounds = - options.per_round - ? std::max(1, syndromes.front().per_round.size()) - : 0; - const std::size_t frames_per_shot = options.per_round ? rounds + 1 : 1; - std::vector> windows; - windows.reserve(num_shots * frames_per_shot); - if (options.per_round) { - for (std::size_t s = 0; s < num_shots; ++s) { - const auto &pr = syndromes[s].per_round; - for (std::size_t r = 0; r < rounds; ++r) { - const std::vector &bits = - (r < pr.size()) ? pr[r] : syndromes[s].measurements; - auto rid = static_cast(s * frames_per_shot + r); - windows.push_back(build_enqueue_frame(bits, rid, /*decoder_id=*/0, - /*counter=*/rid)); - } - auto grid = static_cast(s * frames_per_shot + rounds); - windows.push_back(build_get_corrections_frame( - grid, /*decoder_id=*/0, num_observables, /*reset=*/1)); - } - std::cout << "Per-round mode: " << rounds << " enqueue + 1 get_corrections " - << "per shot (" << frames_per_shot << " frames/shot)\n"; - } else { - for (std::size_t i = 0; i < num_shots; ++i) - windows.push_back(build_rpc_payload(syndromes[i].measurements, - function_id, - static_cast(i))); - } + windows.reserve(num_shots); + for (std::size_t i = 0; i < num_shots; ++i) + windows.push_back(build_rpc_payload(syndromes[i].measurements, function_id, + static_cast(i))); - // Frames may differ in size (per-round); pad all to the largest, 64-aligned. - std::size_t payload_size = 0; - for (const auto &w : windows) - payload_size = std::max(payload_size, w.size()); + std::size_t payload_size = windows.front().size(); std::size_t bytes_per_window = align_up(payload_size, 64); for (auto &window : windows) window.resize(bytes_per_window, 0); std::size_t cycles_per_window = bytes_per_window / 64; - if (cycles_per_window == 0) - cycles_per_window = 1; - - // Per-round multiplies frames by (rounds+1), so all shots rarely fit the - // 512-cycle playback BRAM. Auto-clamp num_shots to what fits (and truncate - // the already-built frames) so the run works without a hand-tuned - // --num-shots; the >120-decode guarantee is owned by the Stage A - // surface_code run, and hardware >120 would need multi-load chunking. - if (options.per_round) { - const std::size_t per_shot_cycles = frames_per_shot * cycles_per_window; - const std::size_t max_shots = - RAM_DEPTH / std::max(1, per_shot_cycles); - if (max_shots == 0) { - std::cerr << "A single shot's " << frames_per_shot << " frames x " - << cycles_per_window << " cycles exceed the " << RAM_DEPTH - << "-cycle playback BRAM\n"; - return 1; - } - if (num_shots > max_shots) { - std::cout << "WARNING: per-round playback is BRAM-limited to " - << max_shots << " shots (requested " << num_shots - << "); truncating.\n"; - num_shots = max_shots; - windows.resize(num_shots * frames_per_shot); - } - } - - const std::size_t num_windows = windows.size(); - if (num_windows * cycles_per_window > RAM_DEPTH) { - std::cerr << "Data exceeds playback BRAM capacity: " << num_windows - << " frames x " << cycles_per_window - << " cycles = " << (num_windows * cycles_per_window) << " > " - << RAM_DEPTH << " depth. Reduce --num-shots" - << (options.per_round ? " (per-round multiplies frames by " - "rounds+1)" - : "") - << ".\n"; + if (num_shots * cycles_per_window > RAM_DEPTH) { + std::cerr << "Data exceeds playback BRAM capacity: " << num_shots + << " shots x " << cycles_per_window + << " cycles = " << (num_shots * cycles_per_window) << " > " + << RAM_DEPTH << " depth\n"; return 1; } - std::cout << "Loaded " << num_shots << " shots / " << num_windows - << " frames (syndrome_size=" << syndrome_size + std::cout << "Loaded " << num_shots + << " shots (syndrome_size=" << syndrome_size << ", payload=" << payload_size << " bytes, padded=" << bytes_per_window << " bytes, " - << cycles_per_window << " cycles/frame)\n"; + << cycles_per_window << " cycles/shot)\n"; // ------------------------------------------------------------------ // Connect to Hololink (or emulator) and reset @@ -1286,9 +980,9 @@ int main(int argc, char **argv) { // ------------------------------------------------------------------ // Configure FPGA SIF registers for RDMA target (if provided) // ------------------------------------------------------------------ - std::uint32_t rdma_num_pages = options.rdma_num_pages.value_or(64); if (options.qp_number && options.rkey && options.buffer_addr) { std::uint32_t rdma_page_size = options.rdma_page_size.value_or(256); + std::uint32_t rdma_num_pages = options.rdma_num_pages.value_or(64); std::cout << "Configuring FPGA SIF for RDMA target:\n" << " QP number: 0x" << std::hex << *options.qp_number @@ -1323,13 +1017,13 @@ int main(int argc, char **argv) { config_write.queue_write_uint32(PLAYER_ADDR + PLAYER_WINDOW_SIZE_OFFSET, static_cast(bytes_per_window)); config_write.queue_write_uint32(PLAYER_ADDR + PLAYER_WINDOW_NUMBER_OFFSET, - static_cast(num_windows)); + static_cast(num_shots)); config_write.queue_write_uint32(PLAYER_ADDR + PLAYER_TIMER_OFFSET, RF_SOC_TIMER_SCALE * options.spacing_us); if (!hololink->write_uint32(config_write)) throw std::runtime_error("Failed to configure player"); - std::cout << "Writing " << num_windows << " windows to playback BRAM..." + std::cout << "Writing " << num_shots << " windows to playback BRAM..." << std::endl; try { write_bram(*hololink, windows, bytes_per_window); @@ -1390,8 +1084,8 @@ int main(int argc, char **argv) { PLAYER_ENABLE)) throw std::runtime_error("Failed to enable player"); - std::cout << "Playback enabled: " << num_shots << " shots / " << num_windows - << " frames on hololink " << options.hololink_ip << "\n"; + std::cout << "Playback enabled: " << num_shots << " shots on hololink " + << options.hololink_ip << "\n"; // ------------------------------------------------------------------ // ILA capture and correction verification @@ -1399,29 +1093,37 @@ int main(int argc, char **argv) { if (options.verify) { std::cout << "\n=== ILA Capture & Verification ===\n"; + // In single-pass mode the player sends exactly num_shots packets, so the + // ILA buffer will not fill completely. Poll until the sample count + // stabilizes (no new samples for 2 consecutive checks). + constexpr int kStableChecks = 2; constexpr int kPollIntervalMs = 500; constexpr int kVerifyTimeoutMs = 30000; - const std::uint32_t expected_samples = static_cast( - std::min(num_windows, ILA_DEPTH)); - std::cout << "Waiting for ILA capture to reach " << expected_samples - << " samples (timeout " << kVerifyTimeoutMs << " ms)...\n"; + std::cout << "Waiting for ILA capture to stabilize (timeout " + << kVerifyTimeoutMs << " ms)...\n"; + std::uint32_t prev_count = 0; + int stable = 0; int elapsed = 0; while (elapsed < kVerifyTimeoutMs) { std::this_thread::sleep_for(std::chrono::milliseconds(kPollIntervalMs)); elapsed += kPollIntervalMs; - const std::uint32_t count = ila_sample_count(*hololink); - if (count >= expected_samples) + std::uint32_t count = ila_sample_count(*hololink); + if (count > 0 && count == prev_count) + ++stable; + else + stable = 0; + prev_count = count; + if (stable >= kStableChecks) break; } std::uint32_t actual_samples = ila_sample_count(*hololink); ila_disable(*hololink); - if (actual_samples < expected_samples) { - std::cerr << "ILA: captured " << actual_samples << " of " - << expected_samples << " expected samples (timeout " - << kVerifyTimeoutMs << " ms)\n"; + if (actual_samples == 0) { + std::cerr << "ILA: captured 0 samples (timeout " << kVerifyTimeoutMs + << " ms)\n"; return 1; } std::cout << "ILA: captured " << actual_samples << " samples\n"; @@ -1432,22 +1134,14 @@ int main(int argc, char **argv) { std::cout << "Read " << samples.size() << " samples from ILA\n"; // Verify correction responses against expected values. - auto vr = verify_captured_responses(samples, syndromes, num_shots, - options.per_round, frames_per_shot, - rdma_num_pages); - - // In per-round mode the response frames split into enqueue ACKs - // (result_len==0) and get_corrections frames (result_len>0); only the - // latter carry corrections. - const std::size_t corrections_returned = vr.rpc_responses - vr.enqueue_acks; + auto vr = verify_captured_responses(samples, syndromes, num_shots); + std::cout << "\n=== Verification Summary ===\n" << " ILA samples captured: " << actual_samples << "\n" << " tvalid=0 (idle): " << vr.tvalid_zero << "\n" - << " RPC response frames: " << vr.rpc_responses << "\n"; - if (options.per_round) - std::cout << " Enqueue ACKs: " << vr.enqueue_acks << "\n" - << " get_corrections frames: " << corrections_returned << "\n"; - std::cout << " Non-RPC frames: " << vr.non_rpc_frames << "\n" + << " RPC requests (syndromes): " << vr.rpc_requests << "\n" + << " RPC responses (corrections): " << vr.rpc_responses << "\n" + << " Non-RPC frames: " << vr.non_rpc_frames << "\n" << " Unique shots verified: " << vr.unique_shots_verified << "\n" << " Corrections matched: " << vr.responses_matched << "\n" @@ -1470,37 +1164,28 @@ int main(int argc, char **argv) { // Print first 5 samples for diagnostic for (std::size_t k = 0; k < 5 && k < vr.latency_samples.size(); ++k) { auto &s = vr.latency_samples[k]; - std::cout << " rid " << std::setw(3) << s.request_id << " (shot " - << s.shot << " local " << s.local << " " - << (s.is_corr ? "get_corrections" : "enqueue") << ")" + std::cout << " Msg " << std::setw(3) << s.msg_id << ": send={sec=" << s.send_sec << ", nsec=" << s.send_nsec << "} recv={sec=" << s.recv_sec << ", nsec=" << s.recv_nsec << "} delta=" << s.delta_ns << " ns\n"; } std::cout << "\n=== PTP Round-Trip Latency ===\n" - << " Samples: " << vr.latency_samples.size() - << " (all captured frames)\n" + << " Samples: " << vr.latency_samples.size() << "\n" << " Min: " << lat_min << " ns\n" << " Max: " << lat_max << " ns\n" << " Avg: " << std::fixed << std::setprecision(1) << lat_avg << " ns\n"; - // One row per captured frame. `kind` is enqueue|get_corrections; - // `local` is the frame index within its shot (per-round). const std::string csv_path = "ptp_latency.csv"; std::ofstream csv(csv_path); if (csv.is_open()) { - csv << "request_id,shot,local,kind,send_sec,send_nsec,recv_sec," - "recv_nsec,delta_ns\n"; + csv << "shot,send_sec,send_nsec,recv_sec,recv_nsec,delta_ns\n"; for (auto &s : vr.latency_samples) - csv << s.request_id << "," << s.shot << "," << s.local << "," - << (s.is_corr ? "get_corrections" : "enqueue") << "," - << s.send_sec << "," << s.send_nsec << "," << s.recv_sec << "," - << s.recv_nsec << "," << s.delta_ns << "\n"; + csv << s.msg_id << "," << s.send_sec << "," << s.send_nsec << "," + << s.recv_sec << "," << s.recv_nsec << "," << s.delta_ns << "\n"; csv.close(); - std::cout << " CSV written: " << csv_path << " (" - << vr.latency_samples.size() << " rows)\n"; + std::cout << " CSV written: " << csv_path << "\n"; } } else { std::cout << "\n PTP latency: no valid timestamps found\n"; diff --git a/libs/qec/unittests/utils/hololink_qldpc_graph_decoder_bridge.cpp b/libs/qec/unittests/utils/hololink_qldpc_graph_decoder_bridge.cpp index 0ec96147e..d9aaf4004 100644 --- a/libs/qec/unittests/utils/hololink_qldpc_graph_decoder_bridge.cpp +++ b/libs/qec/unittests/utils/hololink_qldpc_graph_decoder_bridge.cpp @@ -7,70 +7,42 @@ ******************************************************************************/ /// @file hololink_qldpc_graph_decoder_bridge.cpp -/// @brief QLDPC Relay-BP decoder bridge: Hololink GPU-RoCE ring <-> the -/// self-relaunching device-graph scheduler. +/// @brief QLDPC BP decoder bridge adapter using CPU-launched CUDA graph +/// dispatch (HOST_LOOP) with the generic Hololink bridge skeleton. /// -/// This bridge wires the per-round decode-server protocol onto a real -/// (or emulated) FPGA over RoCE. Unlike the inproc_rpc path -/// (qec_realtime_session, which allocates its own pinned ring), the bridge -/// runs the SAME device-graph scheduler directly on the Hololink DOCA ring: +/// This thin adapter: +/// 1. Parses --config argument (Relay BP config YAML) +/// 2. Loads the decoder config, builds the H tensor, creates the decoder +/// 3. Calls capture_decode_graph() to get a CUDA graph + mailbox +/// 4. Builds a cudaq_function_entry_t with the graph_exec +/// 5. Configures BridgeConfig for HOST_LOOP backend +/// 6. Delegates all Hololink / dispatcher plumbing to bridge_run() /// -/// FPGA --RDMA--> Hololink RX kernel --writes rx_flags--> scheduler graph -/// scheduler graph --DEVICE_CALL append/get/reset; fires decode on a full -/// window (CUDAQ_DISPATCH_STATUS_TRIGGER_GRAPH); tail-self-relaunches--> -/// scheduler writes RPCResponse + tx_flags --> Hololink TX kernel --RDMA--> -/// FPGA +/// The HOST_LOOP dispatcher (CPU thread) polls Hololink ring flags, then +/// launches the CUDA graph for each incoming RPC request. This avoids +/// the 120-outstanding-graph limit of device-side cudaGraphLaunch. /// -/// Flow: -/// 1. Parse --config (Relay BP YAML) + generic bridge args. -/// 2. Create the nv-qldpc decoder; capture_decode_graph(reserved_sms) gives a -/// device-launchable cooperative decode graph and registers the decoder's -/// GpuDecoderState (read by the append/get/reset DEVICE_CALL handlers). -/// 3. Create + start the Hololink transceiver (RX + TX kernels) and adopt its -/// DOCA ring (rx/tx flags + data are GPU pointers). -/// 4. Build a pinned-mapped 3-entry DEVICE_CALL function table (enqueue -/// accumulate, get_corrections, reset_decoder) via the proprietary -/// populate shims (resolved by name -- the bridge exe absorbs -/// libcudaq-qec-realtime-cudevice-proprietary.a and device-links it with -/// the dispatch kernel). -/// 5. Launch the device-graph scheduler on the DOCA ring with the decode -/// graph as its triggered graph, then run the Hololink RX/TX kernels -/// (blocking_monitor) on a worker thread. -/// 6. Run until --timeout or SIGINT, then shut down cleanly. +/// Requires a Grace-based system (DGX Spark / GB200) where GPU memory +/// is CPU-accessible via NVLink-C2C. #include -#include -#include -#include #include -#include -#include #include #include #include #include -#include #include #include -#include "cudaq/realtime/daemon/bridge/hololink/hololink_wrapper.h" #include "cudaq/realtime/hololink_bridge_common.h" #include "cudaq/qec/decoder.h" -#include "cudaq/qec/realtime/decoder_rpc_ids.h" #include "cudaq/qec/realtime/decoding_config.h" #include "cudaq/qec/realtime/graph_resources.h" #include "cudaq/qec/realtime/sparse_to_csr.h" -namespace { - -std::atomic g_stop{false}; -void handle_sigint(int) { g_stop.store(true, std::memory_order_release); } - -constexpr auto kHololinkMonitorStartupGrace = std::chrono::milliseconds(250); - -std::string read_file(const std::string &path) { +static std::string read_file(const std::string &path) { std::ifstream f(path); if (!f.is_open()) { std::cerr << "ERROR: Cannot open file: " << path << std::endl; @@ -79,121 +51,49 @@ std::string read_file(const std::string &path) { return {std::istreambuf_iterator(f), std::istreambuf_iterator()}; } -// Resolve a proprietary DEVICE_CALL populate shim by name and stamp the entry. -// Same dlsym(RTLD_DEFAULT) contract as qec_realtime_session: the symbols are -// exported from this executable because it absorbs the cudevice proprietary -// archive (WHOLE_ARCHIVE) and links with --export-dynamic. -using populate_fn = void (*)(void *); -bool populate_device_call(cudaq_function_entry_t &entry, const char *symbol, - std::uint32_t function_id) { - auto fn = reinterpret_cast(::dlsym(RTLD_DEFAULT, symbol)); - if (!fn) { - std::cerr << "ERROR: dlsym(" << symbol - << ") failed -- the bridge must absorb " - "libcudaq-qec-realtime-cudevice-proprietary.a" - << std::endl; - return false; - } - fn(&entry); - entry.function_id = function_id; - entry.routing_key = 0; - if (entry.dispatch_mode != CUDAQ_DISPATCH_DEVICE_CALL || - !entry.handler.device_fn_ptr) { - std::cerr << "ERROR: " << symbol - << " did not produce a valid DEVICE_CALL entry" << std::endl; - return false; - } - return true; -} - -bool alloc_pinned_mapped(std::size_t bytes, void **host_out, void **dev_out) { - void *h = nullptr; - if (cudaHostAlloc(&h, bytes, cudaHostAllocMapped) != cudaSuccess) - return false; - void *d = nullptr; - if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) { - cudaFreeHost(h); - return false; - } - std::memset(h, 0, bytes); - *host_out = h; - *dev_out = d; - return true; -} - -} // namespace - int main(int argc, char *argv[]) { - namespace rpc = cudaq::qec::decoding::rpc; - std::string config_path; + for (int i = 1; i < argc; i++) { std::string arg = argv[i]; if (arg.find("--config=") == 0) config_path = arg.substr(9); else if (arg == "--help" || arg == "-h") { std::cout - << "Usage: " << argv[0] << " --config=PATH [bridge options]\n\n" - << "QLDPC Relay-BP bridge: Hololink GPU-RoCE ring <-> device-graph " - "scheduler.\n\n" - << " --config=PATH Relay BP config YAML (required)\n" - << " --device=NAME IB device (default: rocep1s0f0)\n" - << " --peer-ip=ADDR FPGA/emulator IP (default: 10.0.0.2)\n" - << " --remote-qp=N Remote QP number (default: 0x2)\n" - << " --gpu=N GPU device ID (default: 0)\n" - << " --timeout=N Timeout seconds (default: 60)\n" - << " --page-size=N Ring slot size (default: 384)\n" - << " --num-pages=N Ring slots (default: 64)\n" - << " --reserved-sms=N SMs reserved for Hololink RX/TX (default: " - "2)\n"; + << "Usage: " << argv[0] << " [options]\n\n" + << "QLDPC BP decoder bridge: Hololink GPU-RoCE <-> HOST_LOOP " + "graph dispatch.\n\n" + << "Decoder options:\n" + << " --config=PATH Path to Relay BP config YAML " + "(required)\n\n" + << "Bridge options (passed to generic skeleton):\n" + << " --device=NAME IB device (default: rocep1s0f0)\n" + << " --peer-ip=ADDR FPGA/emulator IP (default: 10.0.0.2)\n" + << " --remote-qp=N Remote QP number (default: 0x2)\n" + << " --gpu=N GPU device ID (default: 0)\n" + << " --timeout=N Timeout in seconds (default: 60)\n" + << " --page-size=N Ring buffer slot size (default: 384)\n" + << " --num-pages=N Ring buffer slots (default: 64)\n" + << " --exchange-qp Enable QP exchange (emulator mode)\n" + << " --exchange-port=N QP exchange TCP port (default: " + "12345)\n"; return 0; } } + if (config_path.empty()) { std::cerr << "ERROR: --config=PATH is required" << std::endl; return 1; } - cudaq::realtime::BridgeConfig config; - cudaq::realtime::parse_bridge_args(argc, argv, config); - - int reserved_sms = 2; - for (int i = 1; i < argc; i++) { - std::string arg = argv[i]; - if (arg.find("--reserved-sms=") == 0) - reserved_sms = std::stoi(arg.substr(15)); - } - - // Guard: clamp num_pages to the HSB receive/send work-queue depth. - // - // The Hololink gpu_roce_transceiver (HSB 2.6.0-EA2) posts WQE_NUM=64 - // receive/send WQEs and runs one kernel thread per WQE. When the ring is - // deeper than that, a single thread services multiple ring slots (slot t and - // t+64 share one WQE / CQ position), and the free-running RX/TX kernels race - // on that shared resource -- empirically a duplicated frame W plus a dropped - // frame W+64 (verified on the emulator: every failure was an exact (W, W+64) - // pair on one thread, with no RDMA timeouts). A 1:1 slot<->WQE mapping - // (num_pages <= WQE_NUM) is the only safe configuration. We clamp rather - // than abort so a stale/oversized --num-pages can't silently corrupt data. - constexpr unsigned kHsbWqeNum = - 64; // == HSB WQE_NUM (gpu_roce_transceiver_common.hpp) - if (config.num_pages > kHsbWqeNum) { - std::cerr << "WARNING: --num-pages=" << config.num_pages - << " exceeds the HSB transceiver's WQE depth (" << kHsbWqeNum - << "); clamping to " << kHsbWqeNum - << " (a deeper ring multiplexes >1 slot per WQE and races the " - "RX/TX kernels -> duplicate/drop)." - << std::endl; - config.num_pages = kHsbWqeNum; - } - - std::cout << "=== Hololink QLDPC Relay-BP Bridge (device-graph scheduler) ===" + std::cout << "=== Hololink QLDPC BP Decoder Bridge (Graph Launch) ===" << std::endl; - // -- Load decoder config + build the decoder -------------------------------- + // ---- Load decoder config ---- std::string yaml_str = read_file(config_path); if (yaml_str.empty()) return 1; + auto mdc = cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( yaml_str); if (mdc.decoders.empty()) { @@ -202,24 +102,30 @@ int main(int argc, char *argv[]) { } auto &dec = mdc.decoders[0]; + // ---- Build H tensor from sparse representation ---- std::vector h_row_ptr, h_col_idx; - cudaq::qec::realtime::sparse_vec_to_csr(dec.H_sparse, h_row_ptr, h_col_idx); + std::size_t h_rows = cudaq::qec::realtime::sparse_vec_to_csr( + dec.H_sparse, h_row_ptr, h_col_idx); std::size_t bs = dec.block_size; std::size_t ss = dec.syndrome_size; + cudaqx::tensor H_tensor({ss, bs}); for (std::size_t r = 0; r < ss; ++r) for (uint32_t j = h_row_ptr[r]; j < h_row_ptr[r + 1]; ++j) H_tensor.at({r, static_cast(h_col_idx[j])}) = 1; + // ---- Create decoder ---- auto params = dec.decoder_custom_args_to_heterogeneous_map(); auto decoder = cudaq::qec::decoder::get("nv-qldpc-decoder", H_tensor, params); if (!decoder) { std::cerr << "ERROR: Failed to create nv-qldpc-decoder" << std::endl; return 1; } + decoder->set_D_sparse(dec.D_sparse); decoder->set_O_sparse(dec.O_sparse); + // Derive num_measurements and num_observables for frame size calculation std::vector d_rp, d_ci; cudaq::qec::realtime::sparse_vec_to_csr(dec.D_sparse, d_rp, d_ci); std::size_t num_measurements = 0; @@ -230,246 +136,56 @@ int main(int argc, char *argv[]) { std::size_t num_observables = cudaq::qec::realtime::sparse_vec_to_csr(dec.O_sparse, o_rp, o_ci); - std::cout << " block_size=" << bs << " syndrome_size=" << ss - << " num_measurements=" << num_measurements - << " num_observables=" << num_observables << std::endl; + std::cout << " block_size: " << bs << std::endl; + std::cout << " syndrome_size: " << ss << std::endl; + std::cout << " num_measurements: " << num_measurements << std::endl; + std::cout << " num_observables: " << num_observables << std::endl; + (void)h_rows; + // ---- Capture CUDA graph ---- if (!decoder->supports_graph_dispatch()) { std::cerr << "ERROR: nv-qldpc-decoder does not support graph dispatch" << std::endl; return 1; } + cudaq::realtime::BridgeConfig config; + cudaq::realtime::parse_bridge_args(argc, argv, config); + BRIDGE_CUDA_CHECK(cudaSetDevice(config.gpu_id)); - // -- Capture the device-launchable cooperative decode graph ----------------- - // Reserve SMs for the Hololink RX/TX kernels so the cooperative decode can - // still co-reside. This also registers the decoder's GpuDecoderState with - // the proprietary device table (read by the DEVICE_CALL handlers). - void *raw_res = decoder->capture_decode_graph(reserved_sms); + void *raw_res = decoder->capture_decode_graph(/*reserved_sms=*/2); if (!raw_res) { std::cerr << "ERROR: capture_decode_graph() returned null" << std::endl; return 1; } auto *graph_res = static_cast(raw_res); - if (!graph_res->graph_exec) { - std::cerr << "ERROR: capture_decode_graph() produced no graph_exec" - << std::endl; - decoder->release_decode_graph(raw_res); - return 1; - } - std::cout << " Decode graph captured (device-launchable), reserved_sms=" - << reserved_sms << std::endl; - // -- Size the ring frame for the largest per-round RPC ---------------------- - // Per-round protocol: enqueue carries EnqueueRequestPayload + bit-packed - // per-round syndromes; get_corrections returns bit-packed observables. Size - // the page for the worst case (whole-window measurements as an upper bound) - // and let --page-size override upward. - const std::size_t enqueue_max = - sizeof(cudaq::realtime::RPCHeader) + - rpc::align_to_8(sizeof(rpc::EnqueueRequestPayload) + - rpc::bit_packed_bytes(num_measurements)); - const std::size_t get_resp_max = - sizeof(cudaq::realtime::RPCResponse) + - rpc::align_to_8(rpc::bit_packed_bytes(num_observables)); - std::size_t min_frame = std::max(enqueue_max, get_resp_max); - config.frame_size = std::max(config.frame_size, min_frame); - if (config.page_size < config.frame_size) - config.page_size = config.frame_size; - // 128-byte page granularity for Hololink. - config.page_size = (config.page_size + 127) & ~static_cast(127); - std::cout << " frame_size=" << config.frame_size - << " page_size=" << config.page_size - << " num_pages=" << config.num_pages << std::endl; + std::cout << " Graph captured: function_id=0x" << std::hex + << graph_res->function_id << std::dec << std::endl; - // -- Create + start the Hololink transceiver (RX + TX kernels) -------------- - hololink_transceiver_t transceiver = hololink_create_transceiver( - config.device.c_str(), 1, config.remote_qp, config.gpu_id, - config.frame_size, config.page_size, config.num_pages, - config.peer_ip.c_str(), /*forward=*/0, /*rx_only=*/1, /*tx_only=*/1); - if (!transceiver) { - std::cerr << "ERROR: Failed to create Hololink transceiver" << std::endl; - decoder->release_decode_graph(raw_res); - return 1; - } - if (!hololink_start(transceiver)) { - // Do NOT destroy a half-initialized transceiver here: the DOCA teardown - // path double-frees GPU memory that start() never allocated and segfaults, - // which obscures the real error. This is a fatal exit, so let the OS - // reclaim. The most common cause is a missing IPv4-mapped RoCE v2 GID -- - // ensure the bridge netdev has its IPv4 address assigned and is up. - std::cerr << "ERROR: hololink_start failed (often a missing RoCE v2 GID -- " - "check the IB device's IPv4 address is assigned)" - << std::endl; - return 1; - } - BRIDGE_CUDA_CHECK(cudaSetDevice(config.gpu_id)); + // ---- Configure HOST_LOOP bridge ---- + cudaq_function_entry_t entry{}; + entry.handler.graph_exec = graph_res->graph_exec; + entry.function_id = graph_res->function_id; + entry.dispatch_mode = CUDAQ_DISPATCH_GRAPH_LAUNCH; - // -- Adopt the DOCA ring (GPU pointers) ------------------------------------- - auto *rx_data = - reinterpret_cast(hololink_get_rx_ring_data_addr(transceiver)); - auto *rx_flag = hololink_get_rx_ring_flag_addr(transceiver); - auto *tx_data = - reinterpret_cast(hololink_get_tx_ring_data_addr(transceiver)); - auto *tx_flag = hololink_get_tx_ring_flag_addr(transceiver); - if (!rx_data || !rx_flag || !tx_data || !tx_flag) { - std::cerr << "ERROR: null DOCA ring pointer(s)" << std::endl; - hololink_close(transceiver); - hololink_destroy_transceiver(transceiver); - decoder->release_decode_graph(raw_res); - return 1; - } - const std::size_t num_slots = hololink_get_num_pages(transceiver); - const std::size_t slot_size = hololink_get_page_size(transceiver); - - // -- Build the pinned-mapped 3-entry DEVICE_CALL function table ------------- - void *ft_host = nullptr; - void *ft_dev = nullptr; - if (!alloc_pinned_mapped(3 * sizeof(cudaq_function_entry_t), &ft_host, - &ft_dev)) { - std::cerr << "ERROR: function-table alloc failed" << std::endl; - hololink_close(transceiver); - hololink_destroy_transceiver(transceiver); - decoder->release_decode_graph(raw_res); - return 1; - } - auto *entries = static_cast(ft_host); - bool ok = - populate_device_call( - entries[0], - "cudaqx_qec_realtime_dispatch_populate_enqueue_syndromes_device_" - "entry", - rpc::kEnqueueSyndromesFunctionId) && - populate_device_call( - entries[1], - "cudaqx_qec_realtime_dispatch_populate_get_corrections_device_entry", - rpc::kGetCorrectionsFunctionId) && - populate_device_call( - entries[2], - "cudaqx_qec_realtime_dispatch_populate_reset_decoder_device_entry", - rpc::kResetDecoderFunctionId); - if (!ok) { - cudaFreeHost(ft_host); - hololink_close(transceiver); - hololink_destroy_transceiver(transceiver); - decoder->release_decode_graph(raw_res); - return 1; - } + config.dispatch_path = CUDAQ_DISPATCH_PATH_HOST; + config.h_function_entries = &entry; + config.h_func_count = 1; + config.h_mailbox = graph_res->h_mailbox; + config.d_mailbox = graph_res->d_mailbox; - // -- Shutdown flag (pinned-mapped) + stats ---------------------------------- - void *sd_host = nullptr; - void *sd_dev = nullptr; - if (!alloc_pinned_mapped(sizeof(int), &sd_host, &sd_dev)) { - std::cerr << "ERROR: shutdown-flag alloc failed" << std::endl; - cudaFreeHost(ft_host); - hololink_close(transceiver); - hololink_destroy_transceiver(transceiver); - decoder->release_decode_graph(raw_res); - return 1; - } - auto *shutdown_host = static_cast(sd_host); - auto *shutdown_dev = static_cast(sd_dev); - uint64_t *d_stats = nullptr; - BRIDGE_CUDA_CHECK(cudaMalloc(&d_stats, sizeof(uint64_t))); - BRIDGE_CUDA_CHECK(cudaMemset(d_stats, 0, sizeof(uint64_t))); - - // -- Launch the device-graph scheduler on the DOCA ring --------------------- - // Strict-FIFO consumption: shared_ring_mode is OFF. This scheduler is the - // SOLE consumer of the DOCA RX ring (no peer dispatcher), and the Hololink RX - // kernel fills slots strictly in order (window N -> slot N % num_pages). The - // persistent cursor in dispatch_kernel_with_graph keeps current_slot across - // the tail self-relaunch, so the scheduler waits at the next slot in order - // rather than rescanning from 0 -- which is what avoids the slot-reuse race - // (out-of-order grab + flag-clear vs. refill) that shared_ring scanning - // introduced here. The dispatch kernel default is shared-ring OFF, so no - // setter call is needed. - - cudaStream_t sched_stream = nullptr; - BRIDGE_CUDA_CHECK(cudaStreamCreate(&sched_stream)); - - cudaq_dispatch_graph_context *sched_ctx = nullptr; - cudaError_t cerr = cudaq_create_dispatch_graph_regular( - reinterpret_cast(rx_flag), - reinterpret_cast(tx_flag), rx_data, tx_data, - slot_size, slot_size, static_cast(ft_dev), - /*func_count=*/3, /*graph_io_ctx=*/nullptr, shutdown_dev, d_stats, - num_slots, /*num_blocks=*/1, /*threads_per_block=*/64, - graph_res->graph_exec, sched_stream, &sched_ctx); - if (cerr != cudaSuccess) { - std::cerr << "ERROR: cudaq_create_dispatch_graph_regular: " - << cudaGetErrorString(cerr) << std::endl; - cudaFree(d_stats); - cudaFreeHost(ft_host); - cudaFreeHost(sd_host); - hololink_close(transceiver); - hololink_destroy_transceiver(transceiver); - decoder->release_decode_graph(raw_res); - return 1; - } - cerr = cudaq_launch_dispatch_graph(sched_ctx, sched_stream); - if (cerr != cudaSuccess) { - std::cerr << "ERROR: cudaq_launch_dispatch_graph: " - << cudaGetErrorString(cerr) << std::endl; - cudaq_destroy_dispatch_graph(sched_ctx); - cudaFree(d_stats); - cudaFreeHost(ft_host); - cudaFreeHost(sd_host); - hololink_close(transceiver); - hololink_destroy_transceiver(transceiver); - decoder->release_decode_graph(raw_res); - return 1; - } - std::cout << " Scheduler launched (3 DEVICE_CALL entries, triggered decode " - "graph)" - << std::endl; - - // -- Run the Hololink RX/TX kernels on a worker thread ---------------------- - std::signal(SIGINT, handle_sigint); - std::thread monitor([&]() { hololink_blocking_monitor(transceiver); }); - - // blocking_monitor() launches the Hololink RX/TX kernels inside the worker - // thread. Give those launches a short grace period before the orchestration - // script sees "Bridge Ready" and starts playback. - std::this_thread::sleep_for(kHololinkMonitorStartupGrace); - - // Emit the bridge's RDMA target info in the exact format the orchestration - // script greps (extract_hex for QP/Buffer, extract_decimal for RKey) so the - // playback tool can point the FPGA/emulator SIF at our GPU ring, and signal - // readiness AFTER the scheduler + RX/TX kernels are live. - std::cout << "QP Number: 0x" << std::hex - << hololink_get_qp_number(transceiver) << std::dec << "\n"; - std::cout << "RKey: " << hololink_get_rkey(transceiver) << "\n"; - std::cout << "Buffer Addr: 0x" << std::hex - << hololink_get_buffer_addr(transceiver) << std::dec << "\n"; - std::cout << "Bridge Ready" << std::endl; - - std::cout << " Running for up to " << config.timeout_sec - << " s (Ctrl-C to stop)..." << std::endl; - auto deadline = std::chrono::steady_clock::now() + - std::chrono::seconds(config.timeout_sec); - while (!g_stop.load(std::memory_order_acquire) && - std::chrono::steady_clock::now() < deadline) - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - - // -- Shutdown --------------------------------------------------------------- - std::cout << " Stopping..." << std::endl; - __atomic_store_n(shutdown_host, 1, __ATOMIC_RELEASE); - __sync_synchronize(); - cudaStreamSynchronize(sched_stream); // drain the self-relaunch chain - hololink_close(transceiver); // stop RX/TX monitor - if (monitor.joinable()) - monitor.join(); + config.frame_size = sizeof(cudaq::realtime::RPCHeader) + + std::max(num_measurements, num_observables); + if (config.page_size < config.frame_size) + config.page_size = config.frame_size; - cudaq_destroy_dispatch_graph(sched_ctx); - cudaStreamDestroy(sched_stream); - cudaFree(d_stats); - cudaFreeHost(ft_host); - cudaFreeHost(sd_host); - hololink_destroy_transceiver(transceiver); - decoder->release_decode_graph(raw_res); + auto *decoder_ptr = decoder.get(); + config.cleanup_fn = [decoder_ptr, raw_res]() { + decoder_ptr->release_decode_graph(raw_res); + }; - std::cout << "=== Bridge exited cleanly ===" << std::endl; - return 0; + return cudaq::realtime::bridge_run(config); } diff --git a/libs/qec/unittests/utils/hololink_qldpc_graph_decoder_test.sh b/libs/qec/unittests/utils/hololink_qldpc_graph_decoder_test.sh index 55d517319..a72f1648f 100755 --- a/libs/qec/unittests/utils/hololink_qldpc_graph_decoder_test.sh +++ b/libs/qec/unittests/utils/hololink_qldpc_graph_decoder_test.sh @@ -50,15 +50,6 @@ CUDA_QUANTUM_DIR="/workspaces/cuda-quantum" CUDA_QX_DIR="/workspaces/cudaqx" DATA_DIR="" # auto-detected if empty -# Proprietary device-graph artifacts built in the cuda-qx (decode_server1) tree: -# - cudevice archive: enqueue/get/reset DEVICE_CALL handlers + register/ -# populate shims (WHOLE_ARCHIVE-linked + device-linked into the bridge). -# - nv-qldpc plugin .so (dlopen'd; capture_decode_graph builds the decode). -# Override the parent dir with --cuda-qx-priv-dir or each path individually. -CUDA_QX_PRIV_DIR="/workspaces/cuda-qx" -PROPRIETARY_ARCHIVE="${CUDA_QX_PRIV_DIR}/build/lib/libcudaq-qec-realtime-cudevice-proprietary.a" -NV_QLDPC_PLUGIN="${CUDA_QX_PRIV_DIR}/build/lib/decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so" - # Network defaults IB_DEVICE="" # auto-detect BRIDGE_IP="10.0.0.1" @@ -71,13 +62,7 @@ GPU_ID=0 TIMEOUT=60 NUM_SHOTS="" PAGE_SIZE=384 -# Ring depth (num_pages) is intentionally NOT configurable: stock HSB -# (gpu_roce_transceiver, 2.6.0-EA2) posts WQE_NUM=64 receive/send WQEs and one -# thread per WQE, so a ring deeper than 64 makes a single thread service -# multiple slots (slot t and t+64 share a WQE) and races the RX/TX kernels -- -# observed as a duplicated frame W + dropped frame W+64. The bridge and -# playback both default num_pages=64 (1:1 slot<->WQE), which is the only safe -# configuration; the bridge also guards/clamps to 64. +NUM_PAGES=128 SPACING="" CONTROL_PORT=8193 @@ -109,15 +94,8 @@ Build options: (default: /workspaces/holoscan-sensor-bridge) --cuda-quantum-dir DIR cuda-quantum source directory (default: /workspaces/cuda-quantum) - --cuda-qx-dir DIR cudaqx (public) source dir that builds the bridge + - playback (default: /workspaces/cudaqx) - --cuda-qx-priv-dir DIR cuda-qx (proprietary, decode_server1) tree that - provides the cudevice archive + nv-qldpc plugin - (default: /workspaces/cuda-qx); sets the two paths below - --proprietary-archive PATH Prebuilt libcudaq-qec-realtime-cudevice-proprietary.a - (enqueue/get/reset DEVICE_CALL handlers; WHOLE_ARCHIVE- - linked into the bridge) - --nv-qldpc-plugin PATH Prebuilt libcudaq-qec-nv-qldpc-decoder.so (dlopen'd) + --cuda-qx-dir DIR cuda-qx source directory + (default: /workspaces/cuda-qx) --jobs N Parallel build jobs (default: nproc) Network options: @@ -134,6 +112,7 @@ Run options: --no-verify Skip correction verification --num-shots N Limit number of shots --page-size N Ring buffer slot size in bytes (default: 384) + --num-pages N Number of ring buffer slots (default: 128) --spacing N Inter-shot spacing in microseconds (default: 10) --control-port N UDP control port for emulator (default: 8193) @@ -151,13 +130,6 @@ while [[ $# -gt 0 ]]; do --hsb-dir) HSB_DIR="$2"; shift ;; --cuda-quantum-dir) CUDA_QUANTUM_DIR="$2"; shift ;; --cuda-qx-dir) CUDA_QX_DIR="$2"; shift ;; - --cuda-qx-priv-dir) - CUDA_QX_PRIV_DIR="$2" - PROPRIETARY_ARCHIVE="${CUDA_QX_PRIV_DIR}/build/lib/libcudaq-qec-realtime-cudevice-proprietary.a" - NV_QLDPC_PLUGIN="${CUDA_QX_PRIV_DIR}/build/lib/decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so" - shift ;; - --proprietary-archive) PROPRIETARY_ARCHIVE="$2"; shift ;; - --nv-qldpc-plugin) NV_QLDPC_PLUGIN="$2"; shift ;; --jobs) JOBS="$2"; shift ;; --device) IB_DEVICE="$2"; shift ;; --bridge-ip) BRIDGE_IP="$2"; shift ;; @@ -169,6 +141,7 @@ while [[ $# -gt 0 ]]; do --timeout) TIMEOUT="$2"; shift ;; --num-shots) NUM_SHOTS="$2"; shift ;; --page-size) PAGE_SIZE="$2"; shift ;; + --num-pages) NUM_PAGES="$2"; shift ;; --spacing) SPACING="$2"; shift ;; --control-port) CONTROL_PORT="$2"; shift ;; --help|-h) print_usage; exit 0 ;; @@ -285,33 +258,6 @@ setup_port() { _info " Done: $iface is up at $ip" } -# Pre-seed a PERMANENT neighbor entry for a real FPGA on the bridge interface. -# The bridge's QP connect resolves the FPGA's L2 (MAC) address via -# ibv_create_ah, which consults the kernel neighbor table. In FPGA mode the -# setup never primed that table, so the in-call ARP resolution timed out -# (ibv_ah ret=110 -> "Failed to get remote MAC" -> QP connect failure) even -# though the link is up. Ping to force ARP resolution, read the FPGA's MAC, -# and pin it `nud permanent` so the connect resolves immediately. Unlike the -# emulate path (loopback -> both ends share one local MAC), the FPGA's MAC must -# be learned from the wire. -_seed_fpga_neighbor() { - local iface="$1" fpga_ip="$2" - ping -c 3 -W 1 -I "$iface" "$fpga_ip" >/dev/null 2>&1 || true - local mac - mac=$(ip neigh show "$fpga_ip" dev "$iface" 2>/dev/null \ - | awk '{for (i = 1; i <= NF; i++) if ($i == "lladdr") print $(i + 1)}' \ - | head -1) - if [[ -n "$mac" ]]; then - sudo ip neigh replace "$fpga_ip" lladdr "$mac" nud permanent dev "$iface" - _info " Static ARP: $fpga_ip -> $mac on $iface" - else - _err " Could not resolve FPGA MAC for $fpga_ip on $iface." - _err " Check the FPGA is cabled to this NIC, powered, and reachable" - _err " (ping $fpga_ip); otherwise the bridge QP connect will time out" - _err " with 'Failed to get remote MAC'." - fi -} - _add_static_arp() { local local_iface="$1" local remote_ip="$2" @@ -326,51 +272,6 @@ _add_static_arp() { _info " Static ARP: $remote_ip -> $mac on $local_iface" } -# Convert an IPv4 address to the trailing groups of its IPv4-mapped RoCE v2 -# GID, e.g. 10.0.0.1 -> "ffff:0a00:0001". -ipv4_to_gid_suffix() { - local o1 o2 o3 o4 - IFS='.' read -r o1 o2 o3 o4 <<< "$1" - printf "ffff:%02x%02x:%02x%02x" "$o1" "$o2" "$o3" "$o4" -} - -# Poll until the IPv4-mapped RoCE v2 GID for $ip appears on $ib_dev port 1. -# The gpu_roce_transceiver requires this specific GID (subnet_prefix==0, -# interface_id low32==0xFFFF0000); it only exists while the netdev has the -# IPv4 address AND is up, and it populates asynchronously -- so a blind sleep -# races the bridge's hololink_start GID lookup. -wait_for_roce_v2_gid() { - local ib_dev="$1" ip="$2" timeout_s="${3:-15}" - local suffix gids_dir types_dir elapsed=0 - suffix=$(ipv4_to_gid_suffix "$ip") - gids_dir="/sys/class/infiniband/${ib_dev}/ports/1/gids" - types_dir="/sys/class/infiniband/${ib_dev}/ports/1/gid_attrs/types" - if [[ ! -d "$gids_dir" ]]; then - _info " (no GID sysfs for $ib_dev; skipping GID wait)" - return 0 - fi - while (( elapsed < timeout_s * 10 )); do - local g idx gid t - for g in "$gids_dir"/*; do - idx=$(basename "$g") - gid=$(cat "$g" 2>/dev/null) - if [[ "$gid" == *":${suffix}" ]]; then - t=$(cat "${types_dir}/${idx}" 2>/dev/null) - if [[ "$t" == *"RoCE v2"* ]]; then - _info " RoCE v2 GID ready: ${ib_dev}[${idx}] ${gid}" - return 0 - fi - fi - done - sleep 0.1 - elapsed=$((elapsed + 1)) - done - _err "Timed out waiting for IPv4 RoCE v2 GID (${suffix}) on ${ib_dev}." - _err "The bridge's hololink_start will fail GID lookup. Verify ${ip} is" - _err "assigned to the bridge netdev and the interface is up." - return 1 -} - do_setup_network() { _log "Setting up ConnectX network" @@ -428,9 +329,8 @@ do_setup_network() { _add_static_arp "$iface2" "$BRIDGE_IP" "$iface1" fi - # Wait for the bridge device's IPv4 RoCE v2 GID before proceeding so the - # bridge's hololink_start GID lookup can't race GID-table population. - wait_for_roce_v2_gid "$BRIDGE_DEVICE" "$BRIDGE_IP" 15 || true + _info "Waiting 2s for GID tables to populate..." + sleep 2 else local iface_bridge if [[ -n "$IB_DEVICE" ]]; then @@ -447,14 +347,6 @@ do_setup_network() { _info "Bridge interface: $iface_bridge" setup_port "$iface_bridge" "$BRIDGE_IP" "$MTU" BRIDGE_DEVICE=$(netdev_to_ib "$iface_bridge") - - # Wait for the bridge device's IPv4 RoCE v2 GID (same as emulate mode). - wait_for_roce_v2_gid "$BRIDGE_DEVICE" "$BRIDGE_IP" 15 || true - - # Pre-seed the FPGA's neighbor entry so the bridge QP connect can - # resolve its MAC immediately (avoids the ibv_ah timeout / "Failed to - # get remote MAC"). - _seed_fpga_neighbor "$iface_bridge" "$FPGA_IP" fi } @@ -615,21 +507,6 @@ do_build() { return 1 fi - # The device-graph scheduler bridge needs the proprietary cudevice archive - # (enqueue/get/reset DEVICE_CALL handlers + register/populate shims) built - # in the cuda-qx (decode_server1) tree, plus the nv-qldpc plugin .so. These - # are produced outside this script; verify they exist and wire them in. - if [[ ! -f "$PROPRIETARY_ARCHIVE" ]]; then - _err "Proprietary cudevice archive not found: $PROPRIETARY_ARCHIVE" - _err "Build it in cuda-qx (decode_server1): target cudaq-qec-realtime-cudevice-proprietary" - _err "or pass --proprietary-archive PATH." - return 1 - fi - if [[ ! -f "$NV_QLDPC_PLUGIN" ]]; then - _err "nv-qldpc plugin not found: $NV_QLDPC_PLUGIN (build it in cuda-qx)." - return 1 - fi - # Clear stale cmake cache entries (find_library caches NOTFOUND permanently) rm -f "$cuda_qx_build/CMakeCache.txt" @@ -639,8 +516,6 @@ do_build() { -DCMAKE_CUDA_COMPILER="$cuda_compiler" \ -DCUDAToolkit_ROOT="$cuda_toolkit_root" \ -DCUDAQX_QEC_ENABLE_HOLOLINK_TOOLS=ON \ - -DCUDAQ_QEC_BUILD_TRT_DECODER=OFF \ - -DCUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE="$PROPRIETARY_ARCHIVE" \ -DHOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR="$HSB_DIR" \ -DHOLOSCAN_SENSOR_BRIDGE_BUILD_DIR="$hsb_build" \ -DGPU_ROCE_TRANSCEIVER_LIB="$hsb_gpu_roce_lib" \ @@ -651,15 +526,7 @@ do_build() { -DCUDAQ_REALTIME_HOST_DISPATCH_LIBRARY="${cq_build}/lib/libcudaq-realtime-host-dispatch.a" \ -DCUDAQ_REALTIME_BRIDGE_HOLOLINK_LIBRARY="${cq_build}/lib/libcudaq-realtime-bridge-hololink.so" \ -DCUDAQ_INSTALL_PREFIX="${CUDAQ_INSTALL_PREFIX:-/usr/local/cudaq}" \ - -DCUDAQ_DIR="${CUDAQ_INSTALL_PREFIX:-/usr/local/cudaq}/lib/cmake/cudaq" \ 2>&1 | tail -5 - - # The plugin loader searches relative to libcudaq-qec.so; symlink the - # cuda-qx-built nv-qldpc plugin into the cudaqx decoder-plugins dir. - mkdir -p "$cuda_qx_build/lib/decoder-plugins" - ln -sf "$NV_QLDPC_PLUGIN" \ - "$cuda_qx_build/lib/decoder-plugins/$(basename "$NV_QLDPC_PLUGIN")" - cmake --build "$cuda_qx_build" -j "$JOBS" \ --target hololink_qldpc_graph_decoder_bridge \ hololink_fpga_syndrome_playback \ @@ -810,6 +677,7 @@ run_emulated() { --config="$CONFIG_FILE" \ --timeout="$TIMEOUT" \ --page-size="$PAGE_SIZE" \ + --num-pages="$NUM_PAGES" \ > >(tee "$bridge_log") 2>&1 & local bridge_pid=$! PIDS_TO_KILL+=("$bridge_pid") @@ -840,7 +708,6 @@ run_emulated() { _log "Starting syndrome playback (control-port=$CONTROL_PORT)" local playback_args=( --hololink "$EMULATOR_IP" - --per-round --control-port "$CONTROL_PORT" --config "$CONFIG_FILE" --syndromes "$SYNDROMES_FILE" @@ -849,6 +716,7 @@ run_emulated() { --rkey "$bridge_rkey" --buffer-addr "$bridge_addr" --page-size "$PAGE_SIZE" + --num-pages "$NUM_PAGES" ) if $VERIFY; then playback_args+=(--verify) @@ -893,6 +761,7 @@ run_fpga() { --config="$CONFIG_FILE" \ --timeout="$TIMEOUT" \ --page-size="$PAGE_SIZE" \ + --num-pages="$NUM_PAGES" \ > >(tee "$bridge_log") 2>&1 & local bridge_pid=$! PIDS_TO_KILL+=("$bridge_pid") @@ -921,7 +790,6 @@ run_fpga() { _log "Starting syndrome playback (fpga=$FPGA_IP)" local playback_args=( --hololink "$FPGA_IP" - --per-round --config "$CONFIG_FILE" --syndromes "$SYNDROMES_FILE" --function-name nv_qldpc_decode @@ -929,6 +797,7 @@ run_fpga() { --rkey "$bridge_rkey" --buffer-addr "$bridge_addr" --page-size "$PAGE_SIZE" + --num-pages "$NUM_PAGES" ) if $VERIFY; then playback_args+=(--verify) diff --git a/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh b/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh deleted file mode 100755 index 47cd35759..000000000 --- a/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh +++ /dev/null @@ -1,1222 +0,0 @@ -#!/bin/bash -# ============================================================================ # -# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # -# All rights reserved. # -# # -# This source code and the accompanying materials are made available under # -# the terms of the Apache License 2.0 which accompanies this distribution. # -# ============================================================================ # -# -# hsb_fpga_decoding_server_test.sh -# -# Generic orchestration script for end-to-end decoder testing over Holoscan- -# Sensor-Bridge (HSB) RDMA/RoCE, with the decode work served by the standalone -# decoding server (decoding_server) on the CPU HOST_CALL path instead of a -# GPU bridge. Works with any CPU decoder the server's YAML config selects; -# the default profile is pymatching. -# -# Data path: -# FPGA/emulator --RDMA WRITE--> server cpu_roce rx ring -# server --RDMA SEND---> FPGA SIF TX (captured by the ILA) -# -# Division of labor (identical to hololink_qldpc_graph_decoder_test.sh): -# - decoding_server (--transport=cpu_roce --qp_config=hsb_fpga) owns the -# RDMA ring and prints its QP / RKey / Buffer Addr handshake. It performs -# NO Hololink control-plane traffic. -# - hololink_fpga_syndrome_playback is the sole FPGA control-plane writer: -# it programs the SIF RDMA target with the server's handshake values, -# writes the syndrome frames to BRAM, arms the ILA, enables the player, -# and verifies the captured RPC responses. -# -# Modes: -# Default (FPGA): server + playback (requires real FPGA) -# --emulate: emulator + server + playback (no FPGA needed) -# -# Actions (can be combined): -# --build Build the surface_code-4 generator (only) -# --setup-network Configure ConnectX interfaces -# (run is implicit unless only --build / --setup-network are given) -# -# Examples: -# # Full emulated test: build, configure network, run -# ./hsb_fpga_decoding_server_test.sh --emulate --build --setup-network -# -# # Just run (tools already built, network already set up) -# ./hsb_fpga_decoding_server_test.sh --emulate -# -# # Real FPGA -# ./hsb_fpga_decoding_server_test.sh --setup-network --device rocep1s0f0 \ -# --bridge-ip 192.168.0.1 --fpga-ip 192.168.0.2 -# -# Deployment note: -# --build builds ONLY the surface_code-4 generator (the one artifact not -# shipped in the decoding-server image). Everything else -- decoding_server -# (with gpu_roce linked in), the playback tool, the HSB / cudaq-realtime -# shared libs, and the decoder plugins -- is consumed PREBUILT: on a dev rig -# from the /workspaces/*/build trees, and in a productized container from -# their installed locations. This script never builds cuda-quantum, HSB, or -# the decoder server, and needs no proprietary .a at build time. A clean, -# unconfigured rig cannot bootstrap from it. -set -euo pipefail - -# ============================================================================ -# Defaults -# ============================================================================ - -EMULATE=false -DO_BUILD=false -DO_SETUP_NETWORK=false -DO_RUN=true -VERIFY=true - -# Directory defaults. HSB_DIR is used as-is (already on the correct branch); -# CUDA_QUANTUM_DIR should be checked out at the ref in CUDAQX_DIR/.cudaq_version -# (verified with a warning in do_build). -HSB_DIR="/workspaces/holoscan-sensor-bridge" -CUDA_QUANTUM_DIR="/workspaces/cuda-quantum" -CUDAQX_DIR="/workspaces/cudaqx" -DATA_DIR="" # empty => generate data files (see resolve_data_files) - -# Decoder profile. By default the config + syndromes files are GENERATED -# fresh each run by the surface_code-4-yaml binary into GEN_DIR (they are -# derived artifacts and not checked in); --config/--syndromes or --data-dir -# switch to pre-made files and skip generation. -DECODER="pymatching" -CONFIG_FILE="" -SYNDROMES_FILE="" - -# ONNX model for the trt_decoder profile (TRT predecoder + PyMatching global -# decoder). AUTO => generate the tiny identity predecoder at run time -# (output row = [pre_L=0, syndrome untouched], so TRT preserves the syndrome -# and PyMatching performs the actual correction -- no trained model needed). -ONNX_PATH="AUTO" -ONNX_FILE="" - -# Data-generation parameters (surface-code memory experiment). The -# generator's RNG seed is fixed, so runs are reproducible. -GEN_DISTANCE=3 -GEN_ROUNDS=4 -GEN_P_SPAM=0.01 -GEN_SHOTS=100 - -# Server transport. Empty => derived from the decoder profile: -# pymatching -> cpu_roce (HOST_CALL dispatch on the CPU) -# nv-qldpc-decoder -> gpu_roce (self-relaunching device-graph scheduler: -# enqueue/get/reset run as DEVICE_CALLs on the GPU and -# the captured RelayBP decode graph fires device-side) -TRANSPORT="" -# GPU for the gpu_roce scheduler + decode graph. -GPU_ID=0 -# Server-side GPU RoCE ring depth. "auto" chooses a page count whose total -# allocation satisfies the host page-size requirement. -GPU_ROCE_NUM_PAGES=auto - -# Runtime nv-qldpc plugin for the Relay BP profile: the prebuilt -# libcudaq-qec-nv-qldpc-decoder.so, dlopen'd by both the generator (during -# syndrome generation) and the prebuilt decoder server. Not delivered in this -# repo and has no default path -- point at it with --nv-qldpc-plugin or the -# CUDAQ_QEC_NV_QLDPC_PLUGIN env var (eventual home: a GitHub release artifact -# via all_libs_release.yml). The pymatching profile never uses it. The -# proprietary cudevice archive is NOT a concern of this script: it is a -# build-time input to the decoder server, which is consumed prebuilt here. -NV_QLDPC_PLUGIN="${CUDAQ_QEC_NV_QLDPC_PLUGIN:-}" - -# Network defaults -IB_DEVICE="" # auto-detect -BRIDGE_IP="10.0.0.1" # server-side NIC IP (kept the qldpc script's name) -EMULATOR_IP="10.0.0.2" -FPGA_IP="192.168.0.2" -MTU=4096 - -# Run defaults -TIMEOUT=60 -NUM_SHOTS="" -PAGE_SIZE=384 -# CPU RoCE server ring slots. -NUM_SLOTS=64 -# FPGA/emulator playback window pages. -PLAYBACK_NUM_PAGES=512 -# TX SGE bytes for the server's SEND responses. RPCResponse (24B) + a -# bit-packed correction byte fits well inside 64, keeping every response a -# single 512-bit ILA beat. -FRAME_SIZE=64 -SPACING="" -CONTROL_PORT=8193 - -# Build parallelism -JOBS=$(nproc 2>/dev/null || echo 8) - -# ============================================================================ -# Argument Parsing -# ============================================================================ - -print_usage() { - cat <<'EOF' -Usage: hsb_fpga_decoding_server_test.sh [options] - -Generic orchestration script for decoder end-to-end testing over HSB -RDMA/RoCE with the decoding server (decoding_server) on the CPU -HOST_CALL path. Default decoder profile: pymatching. - -Modes: - --emulate Use FPGA emulator (3-tool mode, no FPGA needed) - Default: FPGA mode (2-tool, requires real FPGA) - -Actions: - --build Build all required tools before running - --setup-network Configure ConnectX network interfaces - --no-run Skip running the test (useful with --build) - -Decoder options: - --decoder NAME Decoder profile: pymatching (default), trt_decoder, or - nv-qldpc-decoder (Relay BP). trt_decoder runs a - TensorRT predecoder + PyMatching global decoder in one - server session (requires the TRT plugin, see --build). - By default the config/syndromes files are generated - fresh each run by surface_code-4-yaml into - CUDAQX_DIR/build/hsb_fpga_test_data - --onnx PATH ONNX model for the trt_decoder profile. Default AUTO - generates the identity predecoder at run time (needs - the python3 'onnx' module) - --transport T Server transport: cpu_roce or gpu_roce. Default is - derived from the decoder (pymatching -> cpu_roce, - nv-qldpc-decoder -> gpu_roce device-graph scheduler) - --config PATH Use a pre-made decoding-server YAML config (skips generation) - --syndromes PATH Use a pre-made syndromes text file (skips generation) - --data-dir DIR Use pre-made DIR/config_NAME.yml + DIR/syndromes_NAME.txt - (skips generation) - -Data-generation options (ignored when --config/--syndromes/--data-dir given): - --distance N Surface-code distance (default: 3) - --num-rounds N Measurement rounds (default: 4) - --p-spam F SPAM error probability (default: 0.01) - --gen-shots N Shots to generate in the syndromes file (default: 100) - -Build options: - --hsb-dir DIR holoscan-sensor-bridge source directory - (default: /workspaces/holoscan-sensor-bridge) - --cuda-quantum-dir DIR cuda-quantum source directory; must match the ref in - CUDAQX_DIR/.cudaq_version (default: /workspaces/cuda-quantum) - --cudaqx-dir DIR cudaqx source dir that builds the server + playback - (default: /workspaces/cudaqx) - --nv-qldpc-plugin PATH Prebuilt libcudaq-qec-nv-qldpc-decoder.so, symlinked - into build/lib/decoder-plugins for the prebuilt server - + generator to dlopen. No default; required for the - nv-qldpc profile (or set CUDAQ_QEC_NV_QLDPC_PLUGIN) - --jobs N Parallel build jobs (default: nproc) - -Network options: - --device DEV ConnectX IB device name (default: auto-detect) - --bridge-ip ADDR Server-side NIC IP (default: 10.0.0.1) - --emulator-ip ADDR Emulator IP (default: 10.0.0.2) - --fpga-ip ADDR FPGA IP for non-emulate mode (default: 192.168.0.2) - --mtu N MTU size (default: 4096) - -Run options: - --timeout N Server timeout in seconds (default: 60) - --no-verify Skip correction verification - --num-shots N Limit number of shots - --page-size N Ring buffer slot size in bytes (default: 384) - --frame-size N Server TX SGE bytes, cpu_roce only (default: 64; - gpu_roce uses page-size as HOLOLINK_FRAME_SIZE) - --gpu N GPU device id for gpu_roce (default: 0) - --gpu-roce-num-pages N Server GPU RoCE ring pages (default: auto-align; - starts from playback window pages) - --playback-num-pages N FPGA/emulator playback window pages (default: 512) - --spacing N Inter-shot spacing in microseconds (default: 10) - --control-port N UDP control port for emulator (default: 8193) - - --help, -h Show this help -EOF -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --emulate) EMULATE=true ;; - --build) DO_BUILD=true ;; - --setup-network) DO_SETUP_NETWORK=true ;; - --no-run) DO_RUN=false ;; - --no-verify) VERIFY=false ;; - --decoder) DECODER="$2"; shift ;; - --onnx) ONNX_PATH="$2"; shift ;; - --transport) TRANSPORT="$2"; shift ;; - --gpu) GPU_ID="$2"; shift ;; - --gpu-roce-num-pages) GPU_ROCE_NUM_PAGES="$2"; shift ;; - --playback-num-pages) PLAYBACK_NUM_PAGES="$2"; shift ;; - --nv-qldpc-plugin) NV_QLDPC_PLUGIN="$2"; shift ;; - --config) CONFIG_FILE="$2"; shift ;; - --syndromes) SYNDROMES_FILE="$2"; shift ;; - --data-dir) DATA_DIR="$2"; shift ;; - --distance) GEN_DISTANCE="$2"; shift ;; - --num-rounds) GEN_ROUNDS="$2"; shift ;; - --p-spam) GEN_P_SPAM="$2"; shift ;; - --gen-shots) GEN_SHOTS="$2"; shift ;; - --hsb-dir) HSB_DIR="$2"; shift ;; - --cuda-quantum-dir) CUDA_QUANTUM_DIR="$2"; shift ;; - --cudaqx-dir) CUDAQX_DIR="$2"; shift ;; - --jobs) JOBS="$2"; shift ;; - --device) IB_DEVICE="$2"; shift ;; - --bridge-ip) BRIDGE_IP="$2"; shift ;; - --emulator-ip) EMULATOR_IP="$2"; shift ;; - --fpga-ip) FPGA_IP="$2"; shift ;; - --mtu) MTU="$2"; shift ;; - --timeout) TIMEOUT="$2"; shift ;; - --num-shots) NUM_SHOTS="$2"; shift ;; - --page-size) PAGE_SIZE="$2"; shift ;; - --frame-size) FRAME_SIZE="$2"; shift ;; - --spacing) SPACING="$2"; shift ;; - --control-port) CONTROL_PORT="$2"; shift ;; - --help|-h) print_usage; exit 0 ;; - *) - echo "ERROR: Unknown option: $1" >&2 - print_usage >&2 - exit 1 - ;; - esac - shift -done - -# Derive the transport from the decoder profile unless explicitly chosen. -if [[ -z "$TRANSPORT" ]]; then - case "$DECODER" in - nv-qldpc-decoder) TRANSPORT="gpu_roce" ;; - *) TRANSPORT="cpu_roce" ;; - esac -fi -if [[ "$TRANSPORT" != "cpu_roce" && "$TRANSPORT" != "gpu_roce" ]]; then - echo "ERROR: unknown --transport $TRANSPORT (expected cpu_roce or gpu_roce)" >&2 - exit 1 -fi - -# Some DOCA registrations require the gpu_roce server ring allocation to be -# host-page aligned. Keep playback capacity independent from the server ring, -# and choose a server page count that satisfies the allocation contract. -if [[ "$TRANSPORT" == "gpu_roce" && "$GPU_ROCE_NUM_PAGES" == "auto" ]]; then - HOST_PAGE_SIZE=$(getconf PAGESIZE 2>/dev/null || echo 4096) - SERVER_PAGE_SIZE=$(( ((PAGE_SIZE + 127) / 128) * 128 )) - GPU_ROCE_NUM_PAGES="$PLAYBACK_NUM_PAGES" - while (( (SERVER_PAGE_SIZE * GPU_ROCE_NUM_PAGES) % HOST_PAGE_SIZE != 0 )); do - ((GPU_ROCE_NUM_PAGES++)) - if (( GPU_ROCE_NUM_PAGES > 65536 )); then - echo "ERROR: unable to auto-align gpu_roce ring for page-size=$PAGE_SIZE host-page-size=$HOST_PAGE_SIZE" >&2 - exit 1 - fi - done -fi - -# ============================================================================ -# Logging Helpers -# ============================================================================ - -_log() { echo "==> $*"; } -_info() { echo " $*"; } -_err() { echo "ERROR: $*" >&2; } -_banner() { - echo "" - echo "========================================" - echo " $*" - echo "========================================" - echo "" -} - -# ============================================================================ -# Cleanup -# ============================================================================ - -PIDS_TO_KILL=() -TEMP_FILES=() - -cleanup() { - local pid - for pid in "${PIDS_TO_KILL[@]}"; do - if kill -0 "$pid" 2>/dev/null; then - kill -TERM "$pid" 2>/dev/null || true - sleep 1 - kill -0 "$pid" 2>/dev/null && kill -KILL "$pid" 2>/dev/null || true - fi - done - for f in "${TEMP_FILES[@]}"; do - rm -f "$f" - done -} -trap cleanup EXIT - -# ============================================================================ -# Network Setup (mirrors hololink_qldpc_graph_decoder_test.sh) -# ============================================================================ - -detect_interfaces() { - if ! command -v ibdev2netdev &>/dev/null; then - _err "ibdev2netdev not found. Install rdma-core or Mellanox OFED." - return 1 - fi - ibdev2netdev -} - -ib_to_netdev() { - local ib_dev="$1" - local port="${2:-1}" - ibdev2netdev | awk -v dev="$ib_dev" -v p="$port" \ - '$1 == dev && $3 == p { print $5 }' -} - -netdev_to_ib() { - local iface="$1" - ibdev2netdev | awk -v iface="$iface" '$5 == iface { print $1 }' -} - -setup_port() { - local iface="$1" - local ip="$2" - local mtu="$3" - local ib_dev - - _info "Configuring $iface: ip=$ip mtu=$mtu" - - local other - for other in $(ip -o addr show to "${ip}/24" 2>/dev/null | awk '{print $2}' | sort -u); do - if [[ "$other" != "$iface" ]]; then - _info "Removing stale ${ip}/24 from $other" - sudo ip addr del "${ip}/24" dev "$other" 2>/dev/null || true - fi - done - - sudo ip link set "$iface" up - sudo ip link set "$iface" mtu "$mtu" - sudo ip addr flush dev "$iface" - sudo ip addr add "${ip}/24" dev "$iface" - - ib_dev=$(netdev_to_ib "$iface") - if [[ -n "$ib_dev" ]] && command -v rdma &>/dev/null; then - local port_count - port_count=$(ls -d "/sys/class/infiniband/${ib_dev}/ports/"* 2>/dev/null | wc -l) - for p in $(seq 1 "$port_count"); do - sudo rdma link set "${ib_dev}/${p}" type eth || true - done - _info " RoCEv2 mode configured for $ib_dev" - fi - - if command -v mlnx_qos &>/dev/null; then - sudo mlnx_qos -i "$iface" --trust=dscp 2>/dev/null || true - _info " DSCP trust mode set" - fi - - if command -v ethtool &>/dev/null; then - sudo ethtool -C "$iface" adaptive-rx off rx-usecs 0 2>/dev/null || true - fi - - _info " Done: $iface is up at $ip" -} - -# Pre-seed a PERMANENT neighbor entry for a real FPGA on the server interface. -# The server's QP connect resolves the FPGA's L2 (MAC) address via -# ibv_create_ah, which consults the kernel neighbor table; without priming it -# the in-call ARP resolution times out even though the link is up. -_seed_fpga_neighbor() { - local iface="$1" fpga_ip="$2" - ping -c 3 -W 1 -I "$iface" "$fpga_ip" >/dev/null 2>&1 || true - local mac - mac=$(ip neigh show "$fpga_ip" dev "$iface" 2>/dev/null \ - | awk '{for (i = 1; i <= NF; i++) if ($i == "lladdr") print $(i + 1)}' \ - | head -1) - if [[ -n "$mac" ]]; then - sudo ip neigh replace "$fpga_ip" lladdr "$mac" nud permanent dev "$iface" - _info " Static ARP: $fpga_ip -> $mac on $iface" - else - _err " Could not resolve FPGA MAC for $fpga_ip on $iface." - _err " Check the FPGA is cabled to this NIC, powered, and reachable" - _err " (ping $fpga_ip); otherwise the server QP connect will time out." - fi -} - -_add_static_arp() { - local local_iface="$1" - local remote_ip="$2" - local remote_iface="$3" - local mac - mac=$(ip link show "$remote_iface" | awk '/ether/ {print $2}') - if [[ -z "$mac" ]]; then - _err "Cannot determine MAC address for $remote_iface" - return 1 - fi - sudo ip neigh replace "$remote_ip" lladdr "$mac" nud permanent dev "$local_iface" - _info " Static ARP: $remote_ip -> $mac on $local_iface" -} - -# Convert an IPv4 address to the trailing groups of its IPv4-mapped RoCE v2 -# GID, e.g. 10.0.0.1 -> "ffff:0a00:0001". -ipv4_to_gid_suffix() { - local o1 o2 o3 o4 - IFS='.' read -r o1 o2 o3 o4 <<< "$1" - printf "ffff:%02x%02x:%02x%02x" "$o1" "$o2" "$o3" "$o4" -} - -# Poll until the IPv4-mapped RoCE v2 GID for $ip appears on $ib_dev port 1. -# The CPU RoCE transceiver requires this specific GID; it only exists while -# the netdev has the IPv4 address AND is up, and it populates asynchronously, -# so a blind sleep races the server's GID lookup. -wait_for_roce_v2_gid() { - local ib_dev="$1" ip="$2" timeout_s="${3:-15}" - local suffix gids_dir types_dir elapsed=0 - suffix=$(ipv4_to_gid_suffix "$ip") - gids_dir="/sys/class/infiniband/${ib_dev}/ports/1/gids" - types_dir="/sys/class/infiniband/${ib_dev}/ports/1/gid_attrs/types" - if [[ ! -d "$gids_dir" ]]; then - _info " (no GID sysfs for $ib_dev; skipping GID wait)" - return 0 - fi - while (( elapsed < timeout_s * 10 )); do - local g idx gid t - for g in "$gids_dir"/*; do - idx=$(basename "$g") - gid=$(cat "$g" 2>/dev/null) - if [[ "$gid" == *":${suffix}" ]]; then - t=$(cat "${types_dir}/${idx}" 2>/dev/null) - if [[ "$t" == *"RoCE v2"* ]]; then - _info " RoCE v2 GID ready: ${ib_dev}[${idx}] ${gid}" - return 0 - fi - fi - done - sleep 0.1 - elapsed=$((elapsed + 1)) - done - _err "Timed out waiting for IPv4 RoCE v2 GID (${suffix}) on ${ib_dev}." - _err "The server's cpu_roce bring-up will fail its GID lookup. Verify" - _err "${ip} is assigned to the server netdev and the interface is up." - return 1 -} - -do_setup_network() { - _log "Setting up ConnectX network" - - if $EMULATE; then - local interfaces - interfaces=$(detect_interfaces) - - if [[ -z "$IB_DEVICE" ]]; then - local iface_bridge iface_emulator - local first_dev first_iface second_iface - - first_dev=$(echo "$interfaces" | head -1 | awk '{print $1}') - first_iface=$(echo "$interfaces" | head -1 | awk '{print $5}') - - second_iface=$(echo "$interfaces" | awk -v d="$first_dev" \ - '$1 == d && $3 == 2 {print $5}') - - if [[ -n "$second_iface" ]]; then - iface_bridge="$first_iface" - iface_emulator="$second_iface" - else - second_iface=$(echo "$interfaces" | awk 'NR==2 {print $5}') - if [[ -z "$second_iface" ]]; then - _err "Need two ConnectX ports for emulation mode but only found one." - return 1 - fi - iface_bridge="$first_iface" - iface_emulator="$second_iface" - fi - - _info "Server interface: $iface_bridge" - _info "Emulator interface: $iface_emulator" - setup_port "$iface_bridge" "$BRIDGE_IP" "$MTU" - setup_port "$iface_emulator" "$EMULATOR_IP" "$MTU" - - BRIDGE_DEVICE=$(netdev_to_ib "$iface_bridge") - EMULATOR_DEVICE=$(netdev_to_ib "$iface_emulator") - - _add_static_arp "$iface_bridge" "$EMULATOR_IP" "$iface_emulator" - _add_static_arp "$iface_emulator" "$BRIDGE_IP" "$iface_bridge" - else - # --device accepts either one dual-port device (ports 1+2) or two - # comma-separated single-port devices ("devA,devB": server on devA, - # emulator on devB). - local iface1 iface2 - if [[ "$IB_DEVICE" == *,* ]]; then - local dev1="${IB_DEVICE%%,*}" dev2="${IB_DEVICE#*,}" - iface1=$(ib_to_netdev "$dev1" 1) - iface2=$(ib_to_netdev "$dev2" 1) - if [[ -z "$iface1" || -z "$iface2" ]]; then - _err "Cannot resolve netdevs for devices $dev1 / $dev2" - return 1 - fi - else - iface1=$(ib_to_netdev "$IB_DEVICE" 1) - iface2=$(ib_to_netdev "$IB_DEVICE" 2) - if [[ -z "$iface1" || -z "$iface2" ]]; then - _err "Cannot find two ports on device $IB_DEVICE" - return 1 - fi - fi - setup_port "$iface1" "$BRIDGE_IP" "$MTU" - setup_port "$iface2" "$EMULATOR_IP" "$MTU" - BRIDGE_DEVICE=$(netdev_to_ib "$iface1") - EMULATOR_DEVICE=$(netdev_to_ib "$iface2") - - _add_static_arp "$iface1" "$EMULATOR_IP" "$iface2" - _add_static_arp "$iface2" "$BRIDGE_IP" "$iface1" - fi - - # Wait for the server device's IPv4 RoCE v2 GID before proceeding so - # the server's cpu_roce GID lookup can't race GID-table population. - wait_for_roce_v2_gid "$BRIDGE_DEVICE" "$BRIDGE_IP" 15 || true - else - local iface_bridge - if [[ -n "$IB_DEVICE" ]]; then - iface_bridge=$(ib_to_netdev "$IB_DEVICE" 1) - else - iface_bridge=$(detect_interfaces | head -1 | awk '{print $5}') - fi - - if [[ -z "$iface_bridge" ]]; then - _err "Cannot detect ConnectX interface for the server." - return 1 - fi - - _info "Server interface: $iface_bridge" - setup_port "$iface_bridge" "$BRIDGE_IP" "$MTU" - BRIDGE_DEVICE=$(netdev_to_ib "$iface_bridge") - - wait_for_roce_v2_gid "$BRIDGE_DEVICE" "$BRIDGE_IP" 15 || true - - # Pre-seed the FPGA's neighbor entry so the server QP connect can - # resolve its MAC immediately. - _seed_fpga_neighbor "$iface_bridge" "$FPGA_IP" - fi -} - -# ============================================================================ -# Build -# ============================================================================ - -detect_cuda_arch() { - local max_arch - max_arch=$(nvcc --list-gpu-arch 2>/dev/null \ - | grep -oP 'compute_\K[0-9]+' | sort -n | tail -1) - if [ -n "$max_arch" ]; then - echo "$max_arch" - fi -} - -do_build() { - _log "Building the surface_code-4 generator only (jobs=$JOBS)" - - local cudaqx_build="${CUDAQX_DIR}/build" - - # This script builds ONLY the surface_code-4 generator (the config + - # syndrome producer that is NOT shipped in the decoding-server image). The - # decoder server, playback tool, HSB / cudaq-realtime shared libs, and - # decoder plugins are consumed PREBUILT -- on a dev rig from the - # /workspaces/*/build trees (resolve_paths points there), and in a - # productized image from their installed locations. So a clean, - # unconfigured rig cannot bootstrap from this script. - if [[ ! -f "$cudaqx_build/CMakeCache.txt" ]]; then - _err "cudaqx build dir is not configured ($cudaqx_build/CMakeCache.txt" - _err "missing). This script builds only the surface_code-4 generator and" - _err "consumes the decoder server, playback, libs, and plugins prebuilt." - _err "Configure + build the cudaqx tree once first, or run against a" - _err "prebuilt/installed image." - return 1 - fi - - # cuda-quantum should be at the ref cudaqx pins (the generator links the - # cuda-quantum realtime libs); warn on skew, as the full build did. - local pinned_ref current_ref - pinned_ref=$(jq -r '.cudaq.ref' "${CUDAQX_DIR}/.cudaq_version" 2>/dev/null || true) - current_ref=$(git -C "$CUDA_QUANTUM_DIR" rev-parse HEAD 2>/dev/null || true) - if [[ -n "$pinned_ref" && -n "$current_ref" && "$pinned_ref" != "$current_ref" ]]; then - _err "cuda-quantum checkout ($current_ref) does not match the cudaqx pin" - _err "($pinned_ref) from ${CUDAQX_DIR}/.cudaq_version. Continuing, but the" - _err "realtime libraries may be ABI-skewed against this cudaqx tree." - elif [[ -n "$pinned_ref" ]]; then - _info "cuda-quantum at the cudaqx pin: $pinned_ref" - fi - - # Ensure nvcc is discoverable for the generator's device-code (nvq++) - # compile step; best-effort (a no-op rebuild needs no compiler). - local cuda_compiler="" - if [[ -n "${CMAKE_CUDA_COMPILER:-}" ]]; then - cuda_compiler="${CMAKE_CUDA_COMPILER}" - elif [[ -x /usr/local/cuda/bin/nvcc ]]; then - cuda_compiler="/usr/local/cuda/bin/nvcc" - else - cuda_compiler="$(command -v nvcc || true)" - fi - if [[ -n "$cuda_compiler" && -x "$cuda_compiler" ]]; then - local cuda_bin_dir - cuda_bin_dir="$(dirname "$cuda_compiler")" - case ":$PATH:" in - *":$cuda_bin_dir:"*) ;; - *) export PATH="$cuda_bin_dir:$PATH" ;; - esac - fi - - # Rebuild only the generator. The cudaqx build dir is already configured - # (checked above); ninja regenerates the build if CMakeLists changed - # (reusing the cached configure) and recompiles only what is stale. This - # script never builds cuda-quantum, HSB, or the decoder server. - cmake --build "$cudaqx_build" -j "$JOBS" --target surface_code-4-yaml \ - 2>&1 | tail -5 - _info "surface_code-4 generator built:" - _info " $cudaqx_build/libs/qec/unittests/realtime/app_examples/surface_code-4-yaml" - - _banner "Build complete" -} - -# ============================================================================ -# Tool Path Resolution -# ============================================================================ - -# Decide where the config + syndromes files come from. Pre-made files -# (--config/--syndromes or --data-dir) win and skip generation; otherwise the -# files are generated into GEN_DIR by generate_data_files(). -GENERATE_DATA=false -resolve_data_files() { - GEN_DIR="${CUDAQX_DIR}/build/hsb_fpga_test_data" - - if [[ -n "$DATA_DIR" ]]; then - # Pre-made profile directory (e.g. checked-in data for some decoder). - CONFIG_FILE="${CONFIG_FILE:-${DATA_DIR}/config_${DECODER}.yml}" - SYNDROMES_FILE="${SYNDROMES_FILE:-${DATA_DIR}/syndromes_${DECODER}.txt}" - return 0 - fi - if [[ -n "$CONFIG_FILE" && -n "$SYNDROMES_FILE" ]]; then - return 0 - fi - if [[ -n "$CONFIG_FILE" || -n "$SYNDROMES_FILE" ]]; then - _err "--config and --syndromes must be given together (or use --data-dir)." - return 1 - fi - - # Default: generate both files fresh this run. - GENERATE_DATA=true - CONFIG_FILE="${GEN_DIR}/config_${DECODER}.yml" - SYNDROMES_FILE="${GEN_DIR}/syndromes_${DECODER}.txt" -} - -# Resolve ONNX_FILE for the trt_decoder profile. AUTO generates the tiny -# identity predecoder (mirrors app_examples/surface_code-4-yaml-test.sh): -# output row = [pre_L=0, input syndrome untouched], so TRT preserves the -# syndrome and the PyMatching global decoder performs the actual correction -- -# expected corrections stay computable at generation time without a trained -# model, while still exercising ONNX parse, engine build, and per-shot -# inference. -generate_identity_onnx() { - if [[ "$ONNX_PATH" != "AUTO" ]]; then - if [[ ! -f "$ONNX_PATH" ]]; then - _err "--onnx file not found: $ONNX_PATH" - return 1 - fi - ONNX_FILE="$ONNX_PATH" - return 0 - fi - if ! python3 -c "import onnx" 2>/dev/null; then - _err "python3 module 'onnx' is required to generate the identity ONNX" - _err "model (pip install onnx), or pass a pre-made model with --onnx." - return 1 - fi - ONNX_FILE="${GEN_DIR}/trt_identity_predecoder.onnx" - local syndrome_size=$(((GEN_DISTANCE * GEN_DISTANCE - 1) * GEN_ROUNDS)) - _info "Generating identity ONNX: $ONNX_FILE (syndrome_size=$syndrome_size)" - python3 - "$ONNX_FILE" "$syndrome_size" <<'PY' -import sys - -import onnx -from onnx import TensorProto, helper - -output_path = sys.argv[1] -syndrome_size = int(sys.argv[2]) - -input_info = helper.make_tensor_value_info( - "input", TensorProto.FLOAT, [1, syndrome_size]) -output_info = helper.make_tensor_value_info( - "output", TensorProto.FLOAT, [1, syndrome_size + 1]) -zero = helper.make_node( - "Constant", - [], - ["pre_l"], - value=helper.make_tensor("zero", TensorProto.FLOAT, [1, 1], [0.0]), -) -concat = helper.make_node("Concat", ["pre_l", "input"], ["output"], axis=1) -graph = helper.make_graph( - [zero, concat], "trt_identity_predecoder", [input_info], [output_info]) -model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 19)]) -# IR 9 is sufficient for opset 19 and remains readable by the ONNX checker in -# the CUDA-QX development image. -model.ir_version = 9 -onnx.checker.check_model(model) -onnx.save(model, output_path) -PY - if [[ ! -f "$ONNX_FILE" ]]; then - _err "Identity ONNX generation failed" - return 1 - fi -} - -# Generate the decoder config (DEM + decoder_custom_args) and the syndromes -# file with the surface_code-4-yaml memory-experiment binary. Runs in GEN_DIR -# so the generator's auxiliary outputs land there too. -generate_data_files() { - GENERATOR_BIN="${CUDAQX_DIR}/build/libs/qec/unittests/realtime/app_examples/surface_code-4-yaml" - if [[ ! -x "$GENERATOR_BIN" ]]; then - _err "Data generator not found: $GENERATOR_BIN" - _err "Run with --build to build the tools first." - return 1 - fi - - # trt_decoder profile: the generator needs an ONNX model at - # config-generation time (--onnx_path is required with --decoder_type - # trt_decoder). The TRT plugin itself is consumed PREBUILT and resolved at - # load time from the decoder-plugins dir on LD_LIBRARY_PATH, exactly like - # the pymatching plugin -- no explicit existence check here. - local extra_gen_flags=() - if [[ "$DECODER" == "trt_decoder" ]]; then - mkdir -p "$GEN_DIR" - generate_identity_onnx || return 1 - extra_gen_flags+=(--onnx_path "$ONNX_FILE") - fi - - _log "Generating test data (decoder=$DECODER, distance=$GEN_DISTANCE," \ - "rounds=$GEN_ROUNDS, p_spam=$GEN_P_SPAM, shots=$GEN_SHOTS)" - mkdir -p "$GEN_DIR" - - local gen_ld_path - gen_ld_path="${CUDA_QUANTUM_DIR}/realtime/build/lib:${CUDAQX_DIR}/build/lib" - - # nv-qldpc profile == the Relay BP test: select Relay BP custom args. - local gen_extra_args=() - if [[ "$DECODER" == "nv-qldpc-decoder" ]]; then - gen_extra_args+=(--use-relay-bp) - fi - - _info "$GENERATOR_BIN --distance $GEN_DISTANCE --num_rounds $GEN_ROUNDS" \ - "--p_spam $GEN_P_SPAM --decoder_type $DECODER" \ - "${extra_gen_flags[*]} ${gen_extra_args[*]:-}" \ - "--save_dem $(basename "$CONFIG_FILE")" - (cd "$GEN_DIR" && \ - LD_LIBRARY_PATH="${gen_ld_path}:${LD_LIBRARY_PATH:-}" \ - "$GENERATOR_BIN" \ - --distance "$GEN_DISTANCE" \ - --num_rounds "$GEN_ROUNDS" \ - --p_spam "$GEN_P_SPAM" \ - --decoder_type "$DECODER" \ - ${extra_gen_flags[@]+"${extra_gen_flags[@]}"} \ - ${gen_extra_args[@]+"${gen_extra_args[@]}"} \ - --save_dem "$(basename "$CONFIG_FILE")" > gen_config.log 2>&1) || { - _err "Config generation failed; see ${GEN_DIR}/gen_config.log" - tail -5 "${GEN_DIR}/gen_config.log" >&2 || true - return 1 - } - - # Configuration-identity check: the identity ONNX makes TRT->PyMatching - # output bit-identical to plain PyMatching, so a silently-substituted - # entry would pass verification below. Pin the generated config to the - # combo before anything downstream consumes it. - if [[ "$DECODER" == "trt_decoder" ]]; then - if ! grep -Eq "type:[[:space:]]+trt_decoder" "$CONFIG_FILE" || \ - ! grep -q "onnx_load_path" "$CONFIG_FILE"; then - _err "Generated config lacks a trt_decoder entry with onnx_load_path: $CONFIG_FILE" - return 1 - fi - _info "Config carries the trt_decoder entry (onnx_load_path set)" - fi - - # The server selects its transceiver from the per-decoder `transport:` YAML - # key (default cpu_roce). For gpu_roce, `cuda_device_id` pins graph capture - # and worker-thread execution to the selected GPU. The generator doesn't emit - # these non-default optional fields, so inject them into our generated config - # directly under the decoder's `type:` line. - if [[ "$TRANSPORT" == "gpu_roce" ]]; then - _info "Injecting 'transport: gpu_roce' and cuda_device_id=$GPU_ID into $(basename "$CONFIG_FILE")" - awk -v gpu_id="$GPU_ID" '{ print } - /^[[:space:]]*type:/ && !done { - print " transport: gpu_roce" - print " cuda_device_id: " gpu_id - done = 1 - }' "$CONFIG_FILE" > "${CONFIG_FILE}.tmp" \ - && mv "${CONFIG_FILE}.tmp" "$CONFIG_FILE" - if ! grep -q "transport:.*gpu_roce" "$CONFIG_FILE" || \ - ! grep -q "cuda_device_id:.*$GPU_ID" "$CONFIG_FILE"; then - _err "Failed to inject gpu_roce transport/cuda_device_id into $CONFIG_FILE" - return 1 - fi - fi - - _info "$GENERATOR_BIN --distance $GEN_DISTANCE --num_rounds $GEN_ROUNDS" \ - "--p_spam $GEN_P_SPAM --num_shots $GEN_SHOTS --yaml $(basename "$CONFIG_FILE")" \ - "--save_syndrome $(basename "$SYNDROMES_FILE")" - (cd "$GEN_DIR" && \ - LD_LIBRARY_PATH="${gen_ld_path}:${LD_LIBRARY_PATH:-}" \ - "$GENERATOR_BIN" \ - --distance "$GEN_DISTANCE" \ - --num_rounds "$GEN_ROUNDS" \ - --p_spam "$GEN_P_SPAM" \ - --num_shots "$GEN_SHOTS" \ - --yaml "$(basename "$CONFIG_FILE")" \ - --save_syndrome "$(basename "$SYNDROMES_FILE")" > gen_syndromes.log 2>&1) || { - _err "Syndrome generation failed; see ${GEN_DIR}/gen_syndromes.log" - tail -5 "${GEN_DIR}/gen_syndromes.log" >&2 || true - return 1 - } - - _info "Generated: $CONFIG_FILE" - _info "Generated: $SYNDROMES_FILE" -} - -# The nv-qldpc profile needs the proprietary plugin dlopen-able from the -# cudaqx decoder-plugins dir (used by both the data generator and the server). -# Symlink it opportunistically so plain runs work without --build. -ensure_nv_qldpc_plugin() { - local plugin_dir="${CUDAQX_DIR}/build/lib/decoder-plugins" - # Guard the empty default first: basename "" yields "" so the link path - # would collapse to "$plugin_dir/", which -e reports as an existing - # directory -- a false "already present". - if [[ -z "$NV_QLDPC_PLUGIN" ]]; then - _err "nv-qldpc profile requires the plugin path: pass --nv-qldpc-plugin PATH" - _err "or set CUDAQ_QEC_NV_QLDPC_PLUGIN (eventual home: all_libs_release.yml artifact)." - return 1 - fi - local link="${plugin_dir}/$(basename "$NV_QLDPC_PLUGIN")" - if [[ -e "$link" ]]; then - return 0 - fi - if [[ ! -f "$NV_QLDPC_PLUGIN" ]]; then - _err "nv-qldpc plugin not found: $NV_QLDPC_PLUGIN" - _err "Build it in the proprietary cuda-qx tree or pass --nv-qldpc-plugin PATH." - return 1 - fi - mkdir -p "$plugin_dir" - ln -sf "$NV_QLDPC_PLUGIN" "$link" - _info "nv-qldpc plugin symlinked: $link" -} - -resolve_paths() { - local cudaqx_utils="${CUDAQX_DIR}/build/libs/qec/unittests/utils" - local cq_build_dir="${CUDA_QUANTUM_DIR}/realtime/build/unittests" - - SERVER_BIN="${CUDAQX_DIR}/build/bin/decoding_server" - PLAYBACK_BIN="${cudaqx_utils}/hololink_fpga_syndrome_playback" - EMULATOR_BIN="${cq_build_dir}/utils/hololink_fpga_emulator" - - if [[ ! -x "$SERVER_BIN" ]]; then - _err "Decoding server binary not found: $SERVER_BIN" - _err "Run with --build to build the tools first." - return 1 - fi - if [[ ! -x "$PLAYBACK_BIN" ]]; then - _err "Playback binary not found: $PLAYBACK_BIN" - _err "Run with --build to build the tools first." - return 1 - fi - if $EMULATE && [[ ! -x "$EMULATOR_BIN" ]]; then - _err "Emulator binary not found: $EMULATOR_BIN" - return 1 - fi - if [[ ! -f "$CONFIG_FILE" ]]; then - _err "Config file not found: $CONFIG_FILE" - return 1 - fi - if [[ ! -f "$SYNDROMES_FILE" ]]; then - _err "Syndromes file not found: $SYNDROMES_FILE" - return 1 - fi - - if [ -z "${BRIDGE_DEVICE:-}" ] && [ -n "${IB_DEVICE:-}" ]; then - # Mirror do_setup_network's handling of the comma form ("devA,devB": - # server on devA, emulator on devB) so runs without --setup-network - # split it the same way. - if [[ "$IB_DEVICE" == *,* ]]; then - BRIDGE_DEVICE="${IB_DEVICE%%,*}" - EMULATOR_DEVICE="${IB_DEVICE#*,}" - else - BRIDGE_DEVICE="$IB_DEVICE" - fi - fi - : "${BRIDGE_DEVICE:=rocep1s0f0}" - if $EMULATE; then - : "${EMULATOR_DEVICE:=rocep1s0f1}" - fi -} - -# ============================================================================ -# Output Parsing Helpers -# ============================================================================ - -wait_for_pattern() { - local logfile="$1" - local pattern="$2" - local timeout_sec="$3" - local pid_to_check="${4:-}" - - local poll_ms=500 - local waited_ms=0 - local timeout_ms=$((timeout_sec * 1000)) - while (( waited_ms < timeout_ms )); do - if [[ -n "$pid_to_check" ]] && ! kill -0 "$pid_to_check" 2>/dev/null; then - _err "Process $pid_to_check died unexpectedly" - return 1 - fi - local match - match=$(grep -m1 "$pattern" "$logfile" 2>/dev/null || true) - if [[ -n "$match" ]]; then - echo "$match" - return 0 - fi - sleep 0.5 - waited_ms=$((waited_ms + poll_ms)) - done - _err "Timeout waiting for pattern: $pattern" - return 1 -} - -extract_hex() { - local line="$1" - echo "$line" | grep -oP '0x[0-9a-fA-F]+' | head -1 -} - -extract_decimal() { - local line="$1" - echo "$line" | awk -F': ' '{print $NF}' | tr -d ' ' -} - -# ============================================================================ -# Server + Playback (shared by both modes) -# ============================================================================ - -# Start the decoding server against $1=peer_ip $2=remote_qp; scrape its -# Bridge Ready handshake into SERVER_QP / SERVER_RKEY / SERVER_ADDR. -start_server() { - local peer_ip="$1" remote_qp="$2" server_log="$3" - - _log "Starting decoding server (decoder=$DECODER, transport=$TRANSPORT," \ - "remote-qp=$remote_qp)" - - local server_ld_path - server_ld_path="${CUDA_QUANTUM_DIR}/realtime/build/lib:${CUDAQX_DIR}/build/lib" - - local ready_pattern - if [[ "$TRANSPORT" == "gpu_roce" ]]; then - # Device-graph scheduler path: enqueue/get/reset run as DEVICE_CALLs - # on the GPU and the captured RelayBP decode graph fires device-side. - # The Hololink transceiver is configured via HOLOLINK_* env (the - # server's gpu_roce mode ignores the cpu_roce CLI flags), and eager - # module loading avoids lazy-load stalls inside the persistent - # scheduler (same as the old bridge launcher). - CUDA_MODULE_LOADING=EAGER \ - LD_LIBRARY_PATH="${server_ld_path}:${LD_LIBRARY_PATH:-}" \ - HOLOLINK_DEVICE="$BRIDGE_DEVICE" \ - HOLOLINK_PEER_IP="$peer_ip" \ - HOLOLINK_REMOTE_QP="$((remote_qp))" \ - HOLOLINK_FRAME_SIZE="$PAGE_SIZE" \ - HOLOLINK_NUM_PAGES="$GPU_ROCE_NUM_PAGES" \ - "$SERVER_BIN" \ - --config="$CONFIG_FILE" \ - --transport=gpu_roce \ - --timeout="$TIMEOUT" \ - > >(tee "$server_log") 2>&1 & - # The GpuRoceTransceiver prints the QP/RKey/Buffer handshake during - # server construction, BEFORE this READY sentinel -- so waiting for - # READY guarantees the three lines are scrapeable. - ready_pattern="QEC_DECODING_SERVER_READY gpu_roce" - else - LD_LIBRARY_PATH="${server_ld_path}:${LD_LIBRARY_PATH:-}" \ - "$SERVER_BIN" \ - --config="$CONFIG_FILE" \ - --transport=cpu_roce \ - --qp_config=hsb_fpga \ - --device="$BRIDGE_DEVICE" \ - --peer-ip="$peer_ip" \ - --remote-qp="$remote_qp" \ - --num-slots="$NUM_SLOTS" \ - --slot-size="$PAGE_SIZE" \ - --frame-size="$FRAME_SIZE" \ - --timeout="$TIMEOUT" \ - > >(tee "$server_log") 2>&1 & - ready_pattern="Bridge Ready" - fi - SERVER_PID=$! - PIDS_TO_KILL+=("$SERVER_PID") - _info "Server PID: $SERVER_PID" - - wait_for_pattern "$server_log" "$ready_pattern" 60 "$SERVER_PID" >/dev/null || { - _err "Decoding server did not become ready" - _err "--- Server log ---" - cat "$server_log" >&2 - return 1 - } - - # Configuration-identity check (all profiles): the server names the - # decoder session it actually constructed. Matters most for trt_decoder, - # where the identity ONNX makes a silently-substituted pymatching session - # pass the corrections verification bit-identically. - wait_for_pattern "$server_log" "decoder 0 type: ${DECODER}" 5 "$SERVER_PID" >/dev/null || { - _err "Server did not construct a '${DECODER}' decoder session" - _err "--- Server log ---" - cat "$server_log" >&2 - return 1 - } - - local qp_line rkey_line addr_line - qp_line=$(wait_for_pattern "$server_log" "QP Number:" 5 "$SERVER_PID") || return 1 - rkey_line=$(wait_for_pattern "$server_log" "RKey:" 5 "$SERVER_PID") || return 1 - addr_line=$(wait_for_pattern "$server_log" "Buffer Addr:" 5 "$SERVER_PID") || return 1 - - SERVER_QP=$(extract_hex "$qp_line") - SERVER_RKEY=$(extract_decimal "$rkey_line") - SERVER_ADDR=$(extract_hex "$addr_line") - - _info "Server QP: $SERVER_QP" - _info "Server RKey: $SERVER_RKEY" - _info "Server Buffer: $SERVER_ADDR" -} - -# Run playback against $1=hololink_ip; extra args appended from $2... -run_playback() { - local hololink_ip="$1"; shift - - _log "Starting syndrome playback (hololink=$hololink_ip)" - local playback_args=( - --hololink "$hololink_ip" - --per-round - --config "$CONFIG_FILE" - --syndromes "$SYNDROMES_FILE" - --qp-number "$SERVER_QP" - --rkey "$SERVER_RKEY" - --buffer-addr "$SERVER_ADDR" - --page-size "$PAGE_SIZE" - --num-pages "$PLAYBACK_NUM_PAGES" - "$@" - ) - if $VERIFY; then - playback_args+=(--verify) - fi - if [[ -n "$NUM_SHOTS" ]]; then - playback_args+=(--num-shots "$NUM_SHOTS") - fi - if [[ -n "$SPACING" ]]; then - playback_args+=(--spacing "$SPACING") - fi - - local playback_rc=0 - "$PLAYBACK_BIN" "${playback_args[@]}" || playback_rc=$? - return $playback_rc -} - -# ============================================================================ -# Run: Emulated Mode (3 tools) -# ============================================================================ - -run_emulated() { - _banner "Decoding Server Decode Loop Test (Emulated FPGA, $DECODER)" - - local emu_log server_log - emu_log=$(mktemp /tmp/hsb_decoding_server_emulator.XXXXXX.log) - server_log=$(mktemp /tmp/hsb_decoding_server.XXXXXX.log) - TEMP_FILES+=("$emu_log" "$server_log") - - # ---- 1. Start emulator ---- - _log "Starting FPGA emulator on port $CONTROL_PORT" - "$EMULATOR_BIN" \ - --device="$EMULATOR_DEVICE" \ - --port="$CONTROL_PORT" \ - --bridge-ip="$BRIDGE_IP" \ - --page-size="$PAGE_SIZE" \ - --num-pages="$PLAYBACK_NUM_PAGES" \ - > >(tee "$emu_log") 2>&1 & - local emu_pid=$! - PIDS_TO_KILL+=("$emu_pid") - _info "Emulator PID: $emu_pid" - - local emu_qp_line - emu_qp_line=$(wait_for_pattern "$emu_log" "Emulator QP:" 30 "$emu_pid") || { - _err "Failed to get emulator QP number" - return 1 - } - local emu_qp - emu_qp=$(extract_hex "$emu_qp_line") - _info "Emulator QP: $emu_qp" - - # ---- 2. Start decoding server ---- - start_server "$EMULATOR_IP" "$emu_qp" "$server_log" || return 1 - - # ---- 3. Start playback tool ---- - run_playback "$EMULATOR_IP" --control-port "$CONTROL_PORT" -} - -# ============================================================================ -# Run: FPGA Mode (2 tools) -# ============================================================================ - -run_fpga() { - _banner "Decoding Server Decode Loop Test (Real FPGA, $DECODER)" - - local server_log - server_log=$(mktemp /tmp/hsb_decoding_server.XXXXXX.log) - TEMP_FILES+=("$server_log") - - # ---- 1. Start decoding server (FPGA data-plane QP is fixed 0x2) ---- - start_server "$FPGA_IP" "0x2" "$server_log" || return 1 - - # ---- 2. Start playback tool (BOOTP enumeration; no control port) ---- - run_playback "$FPGA_IP" -} - -# ============================================================================ -# Main -# ============================================================================ - -main() { - _banner "HSB FPGA Decoding Server Test" - - _info "Decoder: $DECODER (decoding server, CPU HOST_CALL path)" - if $EMULATE; then - _info "Mode: FPGA Emulation (3-tool)" - else - _info "Mode: Real FPGA (2-tool)" - fi - echo "" - - # ---- Build ---- - if $DO_BUILD; then - do_build - fi - - # ---- Network setup ---- - if $DO_SETUP_NETWORK; then - do_setup_network - fi - - # ---- Run ---- - if ! $DO_RUN; then - _log "Skipping test run (--no-run)" - return 0 - fi - - if [[ "$DECODER" == "nv-qldpc-decoder" ]]; then - ensure_nv_qldpc_plugin - fi - resolve_data_files - if $GENERATE_DATA; then - generate_data_files - fi - resolve_paths - - local rc=0 - if $EMULATE; then - run_emulated || rc=$? - else - run_fpga || rc=$? - fi - - # ---- Verdict ---- - echo "" - if [[ $rc -eq 0 ]]; then - _banner "DECODING SERVER DECODE LOOP ($DECODER): PASS" - else - _banner "DECODING SERVER DECODE LOOP ($DECODER): FAIL" - fi - - return $rc -} - -main diff --git a/libs/solvers/lib/optimizers/lbfgs/CMakeLists.txt b/libs/solvers/lib/optimizers/lbfgs/CMakeLists.txt index eeb278ed0..eccfd36e1 100644 --- a/libs/solvers/lib/optimizers/lbfgs/CMakeLists.txt +++ b/libs/solvers/lib/optimizers/lbfgs/CMakeLists.txt @@ -16,14 +16,7 @@ if(NOT liblbfgs_POPULATED) set(LBFGS_USE_SSE ON) set(LBFGS_lib_TARGET_NAME liblbfgs) set(LBFGS_BUILD_SHARED_LIBS OFF) - block() - # libLBFGS still declares cmake_minimum_required(VERSION 2.8.12). - # CMake 4 removed compatibility with versions below 3.5, so keep this - # policy floor scoped to the vendored dependency. - set(CMAKE_POLICY_VERSION_MINIMUM 3.5) - add_subdirectory(${liblbfgs_SOURCE_DIR} ${liblbfgs_BINARY_DIR} - EXCLUDE_FROM_ALL) - endblock() + add_subdirectory(${liblbfgs_SOURCE_DIR} ${liblbfgs_BINARY_DIR} EXCLUDE_FROM_ALL) endif() # ============================================================================== diff --git a/scripts/build_wheels.sh b/scripts/build_wheels.sh index 56681fa70..8e222a92c 100755 --- a/scripts/build_wheels.sh +++ b/scripts/build_wheels.sh @@ -8,7 +8,7 @@ # the terms of the Apache License 2.0 which accompanies this distribution. # # ============================================================================ # -IMAGE_NAME=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-amd64-cu12.6-gcc12-main +IMAGE_NAME=ghcr.io/nvidia/cuda-quantum-devdeps:manylinux-amd64-cu12.6-gcc12-cudaqx-0.15.0 CONTAINER_NAME=cudaqx_wheel_builder CONTAINER_STATUS=$(docker container inspect -f '{{.State.Status}}' $CONTAINER_NAME 2>/dev/null)