diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e62da101..a6a3e5ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,27 @@ jobs: uv sync --locked --python "${PYTHON_VERSION}" --extra dev --extra torch --extra viz --extra photometry --extra ximage --extra rubin --extra catalog - run: make ci-test-cpu + - name: Build and test native xBeam CPU backend + if: matrix.name == 'linux-x64' && matrix.python == '3.12' + run: | + cmake -S src/cuphoton/xbeam/native -B build/xbeam-native -DCMAKE_BUILD_TYPE=Release + cmake --build build/xbeam-native -j2 + uv sync --locked --group native-build --inexact + CUPHOTON_XBEAM_BUILD_EXT=1 uv run --no-sync \ + python setup.py build_ext --build-lib build/native-binding + XBEAM_NATIVE_BINDING_DIR=build/native-binding uv run --no-sync pytest -q -rs \ + --junitxml="${RUNNER_TEMP}/xbeam-native-cpu.xml" tests/xbeam/native + uv run --no-sync python - <<'PYTEST' + import os + from pathlib import Path + import xml.etree.ElementTree as ET + + report = ET.parse(Path(os.environ["RUNNER_TEMP"]) / "xbeam-native-cpu.xml") + assert report.findall(".//testcase"), "Native xBeam tests must execute" + assert not report.findall(".//skipped"), "Native xBeam tests must not skip" + assert not report.findall(".//failure") + assert not report.findall(".//error") + PYTEST - name: Build native xRay CPU extension if: matrix.name == 'linux-x64' && matrix.python == '3.12' env: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 133669fd..566e918c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,6 +39,7 @@ repos: rev: v22.1.8 hooks: - id: clang-format + exclude: ^src/cuphoton/xbeam/native/third_party/ # identify classifies both .cu and .cuh as cuda. types_or: [c++, c, cuda] diff --git a/MANIFEST.in b/MANIFEST.in index 4f96d850..6989e746 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -33,8 +33,11 @@ prune dev recursive-include examples *.py *.yaml recursive-include examples/distributed-pipeline *.json *.txt *.license recursive-include tests *.py +recursive-include tests/xbeam/native *.cpp recursive-include src/cuphoton/xdr/src *.cpp *.h *.sh recursive-include src/cuphoton/xfit/src *.cpp *.cu *.cuh *.h recursive-include src/cuphoton/core/src *.cpp *.cu *.h recursive-include src/cuphoton/xray/src *.cpp *.cu *.h recursive-include src/cuphoton/xbeam/schemas *.json + +recursive-include src/cuphoton/xbeam/native *.pyi *.pyx *.hpp *.cpp *.cu *.py CMakeLists.txt LICENSE.MIT diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 39ed4dfd..401e46a7 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -465,3 +465,10 @@ DragonHPC, `mpi4py`, MPI, and CUDA runtime libraries are installed separately. A development source build links against the available CFITSIO and CUDA libraries. Distributors must retain the licenses and notices for the libraries included in their artifacts. + +## nlohmann/json (native xBeam) + +Version 3.12.0, copyright Niels Lohmann, distributed under the MIT License. +The pinned single-header source and complete license are included under +`src/cuphoton/xbeam/native/third_party/nlohmann/`. +Upstream: https://github.com/nlohmann/json/tree/v3.12.0 diff --git a/dev/uv.lock b/dev/uv.lock index 8736ddb6..fdf9726d 100644 --- a/dev/uv.lock +++ b/dev/uv.lock @@ -205,6 +205,7 @@ docs = [ { name = "torch", specifier = ">=2.13,<3", index = "https://download.pytorch.org/whl/cpu", conflict = { package = "cuphoton", group = "docs" } }, ] native-build = [ + { name = "cython", specifier = ">=3.1,<4" }, { name = "pybind11", specifier = ">=3.0,<4" }, { name = "setuptools", specifier = ">=84.2.1" }, { name = "setuptools-scm", specifier = "==10.3.4" }, diff --git a/docs/xbeam/README.md b/docs/xbeam/README.md index 0fbfdf1d..fa3203b3 100644 --- a/docs/xbeam/README.md +++ b/docs/xbeam/README.md @@ -38,7 +38,7 @@ lag conventions, validity masks and independent result shards. A radio job describes station clocks, spectral bands, baseline products, scans and their delay models. The radio adapter reads two-bit, non-legacy -VDIF files. Pass a normalized JSON job or supported DiFX `.input` metadata. +VDIF files. Pass a normalized JSON job or supported text `.input` metadata. Correlation reads each recording through a header index (a sidecar). Build a sidecar for every recording first with `build-vdif-sidecar`, or pass `--assume-ordered` for recordings you have @@ -133,3 +133,9 @@ batches, with asynchronous uploads for GPU execution. Automatic limits preserve the backend defaults. The radio command still accepts `--vdif-upload-workers` as an alias. See [time-series input](time-series.md) for HDF5 queue and worker bounds. + +## Experimental native radio backend + +An optional [C++/CUDA backend](native.md) provides standalone radio execution +and a Cython entry point. It is built separately; the existing Python backends +and their defaults are unchanged. diff --git a/docs/xbeam/native.md b/docs/xbeam/native.md new file mode 100644 index 00000000..51cb1de4 --- /dev/null +++ b/docs/xbeam/native.md @@ -0,0 +1,233 @@ +# Experimental native backend + +The C++17/CUDA execution library lives under `src/cuphoton/xbeam/native`. +The standalone radio runner needs neither Python nor CuPy at runtime. The +HDF5 adapter uses Python for metadata and result serialization while keeping +raw reads and numerical execution in C++. Support is experimental. The optional +Cython interface exposes the experimental `native-correlate` command; +it is not a production `cuphoton xbeam correlate` backend. + +Build the standalone runtime with a CUDA toolkit, OpenSSL development files, +and a C++17 compiler: + +```bash +cmake -S src/cuphoton/xbeam/native -B build/xbeam-native \ + -DCMAKE_BUILD_TYPE=Release \ + -DXBEAM_ENABLE_CUDA=ON \ + -DXBEAM_ENABLE_CUFILE=ON \ + -DCMAKE_CUDA_ARCHITECTURES=native +cmake --build build/xbeam-native -j +``` + +Python is used during the build to extract the existing cuPhoton-owned CUDA +kernel sources. Execution uses AOT kernels and cuFFT. With cuFile enabled, its +headers and library must be available in the CUDA toolkit. Omit that option +for a POSIX-only build. + +Run an explicitly bounded integration window: + +```bash +build/xbeam-native/xbeam-native-correlate JOB.input \ + --output NEW_OUTPUT_DIRECTORY \ + --index-dir SIDECARS \ + --scan SCAN_ID \ + --first-integration 0 \ + --integration-count 1 \ + --fft-tile 400 \ + --device 0 \ + --vdif-io cufile \ + --profile interop-real-usb-buffered-window-replay-v1 +``` + +The runner loads `.input`, `.calc`, and `.im` metadata; validates existing +schema-v4 sidecars; computes buffered-window source selection, timing and +validity; reads physical frames; performs rotation, FFT, channelization and +XMAC; accumulates integrations; and writes a SWIN file plus `native-run.json`. +The output directory must be new. A failed run has no completion report. +A contiguous run writes `visibilities.swin`; dynamically scheduled runs write +`visibilities.chunk-NNNNNN.swin` files, listed in the completion report. +Choose `--vdif-io posix` for reads through pinned host memory. Both delivery +paths use the same GPU frame-payload extraction and correlation kernels. +cuFile sessions, file handles and device registrations persist across reads. +Direct reads and their owning GPU buffer views use 64-KiB alignment. Each +read chunk has its own persistent registration so delivery does not depend +on offsets into a larger registered region. +`--upload-workers` bounds concurrent reads for either I/O mode. +`--read-chunk-bytes` sets their maximum size (16 MiB by default), in +64-KiB multiples. More workers or larger reads can increase contention when +several GPUs share storage; tune these limits for the machine. +The standalone runner also supports `--read-staging shared` for cuFile. +It retains one registered raw staging buffer per upload worker and extracts +frame payloads before reusing that buffer, including payloads split across +read boundaries. The default `per-input` mode retains raw buffers with each +input slot. Both produce the same packed payload and validity buffers. +`--prefetch on` uses two input slots to overlap the next read with current +GPU work; `--prefetch off` provides a synchronous comparison. Both preserve +the same source selection and numerical contract. +XMAC packs products that share spectra, using their addresses and strides; +output positions and per-product normalization and weight rules are retained. +Failed or incomplete reads cannot publish an input buffer. Chunk tasks drain +before their input allocations are released. + +Radio execution requires one real-sampled physical recording per datastream, +a shared FFT time grid, whole subintegrations per output integration, and +whole reader requests per tile. The profile remains explicit; unsupported +geometry fails rather than delegating to Python or substituting a profile. +Parallel-hand autocorrelations currently use the interoperability truncation +convention. Additional processing profiles, JSON-job loading, general sampling +geometry and shard publication remain open. +The optional CPU inspection binding builds with `CUPHOTON_XBEAM_BUILD_EXT=1`. +The default cuPhoton package build remains pure Python. + +Inspection commands: + +```bash +build/xbeam-native/xbeam-native inspect-vdif RECORDING.vdif +build/xbeam-native/xbeam-native inspect-job JOB.input +build/xbeam-native/xbeam-native inspect-index RECORDING.vdif INDEX.vdix +``` + +Component executables `xbeam-native-station-test` and +`xbeam-native-products-test` check CUDA execution. The bounded +`xbeam-native-station-file-test` diagnoses station frontend output; it does +not generate baseline visibilities. Component checks do not replace full-scan numerical and performance +qualification for a particular input and execution configuration. + +To build the optional CUDA Cython interface, set +`CUPHOTON_XBEAM_BUILD_EXT=1 CUPHOTON_XBEAM_NATIVE_CUDA=1` during installation. +This binding links cuFFT and cuFile, so both development libraries are required. +`CUDA_HOME` selects the toolkit, and +`CUPHOTON_XBEAM_CUDA_ARCHITECTURES` can select explicit CMake CUDA targets for +builds without an accessible GPU. Then the same execution library is available +without launching a subprocess: + +```python +from cuphoton.xbeam.native import correlate + +report = correlate( + "JOB.input", "NEW_OUTPUT_DIRECTORY", "SIDECARS", "SCAN_ID", + first_integration=0, integration_count=1, fft_tile=1250, + device=0, vdif_io="cufile", + profile="interop-real-usb-buffered-window-replay-v1", +) +``` + +The CPU-only extension explicitly refuses correlation. It never silently +falls back to Python execution. +Python callers must supply `integration_count` and `profile` explicitly. +Input delivery defaults to buffered POSIX reads. Each call runs its selected +window to completion; use bounded windows for interactive applications. + +With the CUDA extension installed, the command line calls the same Cython +entry point without entering Python radio loading, planning, or execution: + +```bash +cuphoton xbeam native-correlate JOB.input \ + --output NEW_OUTPUT_DIRECTORY --index-dir SIDECARS \ + --scan SCAN_ID --first-integration 0 --integration-count 8 \ + --vdif-io cufile --fft-tile 1250 --upload-workers 2 \ + --profile interop-real-usb-buffered-window-replay-v1 +``` + +Use the standalone executable when no Python runtime is desired. Native +outputs currently require separate shard assembly; they do not yet publish +the manifests required by `merge-shards`. + +## CPU development checks + +The CPU parser, reader planner, codec, and Cython inspection binding can be +checked without a GPU. Install the development extra, CMake, OpenSSL headers, +and a C++17 compiler, then run from a source checkout: + +```bash +python -m pip install Cython setuptools-scm +cmake -S src/cuphoton/xbeam/native -B build/xbeam-native \ + -DCMAKE_BUILD_TYPE=Release +cmake --build build/xbeam-native -j +CUPHOTON_XBEAM_BUILD_EXT=1 python setup.py build_ext --build-lib build/native-binding +XBEAM_NATIVE_BINDING_DIR=build/native-binding python -m pytest tests/xbeam/native +``` + +Native binding tests are skipped when the optional binding is not supplied. +The CLI routing tests also run in a pure Python installation. Default package +builds do not require a CUDA toolkit or OpenSSL development files. + +## Scheduling and I/O + +For buffered POSIX, prefer contiguous integration windows per GPU to preserve +read locality. Budget CPU cores for all GPU workers on the node; limiting +four workers to one small CPU/NUMA subset can dominate input delivery time. +The standalone runner additionally offers `--vdif-io posix-direct`, which +uses aligned CPU `O_DIRECT` reads and pinned host-to-device copies. This is +separate from cuFile/GDS and is not automatically selected for POSIX. + +Outputs should be written to node-local scratch before publication to shared +storage. Numerical acceptance should compare the complete selected integration +scope; output assembly and comparison are currently external to this backend. + +The standalone runner can distribute one integration scope dynamically with +`--claim-dir DIRECTORY --workers-total N --min-chunk COUNT`. Create an empty +claim directory for that scope, give every worker the same job, scan, +integration range and scheduling options, and give each worker its own new +output directory. Workers claim chunks atomically and list their completed +files in `native-run.json`. Assembly must verify complete, nonoverlapping +coverage across all worker reports. + +Claims remain assigned if a worker fails. Treat that campaign as incomplete +and use fresh claim and output directories when rerunning its scope. Claim +directories belong to one campaign and its common configuration. + +## HDF5 time series + +The native engine also computes generic time-series spectra and lag +correlations. HDF5 metadata is validated by the existing Python adapter; +C++ owns raw file reads, pinned/device buffers, cuFFT plans, cross-products, +validity statistics, and inverse transforms. Python serializes the owned +host results through the existing result writer. This path requires the +CUDA Cython extension and does not require CuPy at runtime. + +Use the existing series configuration with `--backend native`: + +```bash +cuphoton xbeam correlate-series series.json \ + --backend native --output results.h5 --upload-workers 2 +``` + +```json +{ + "input": "signals.h5", + "datasets": {"a": "/signals/a", "b": "/signals/b"}, + "io_backend": "posix", + "correlation": { + "pairs": [["a", "b"]], + "window_samples": 4096, + "hop_samples": 2048, + "windows_per_result": 16, + "batch_windows": 8, + "output": "both", + "lag_mode": "linear" + } +} +``` + +Both `posix` and `cufile` use the same native numerical pipeline and the +existing time-series result schema. cuFile requires the existing aligned +HDF5 raw-layout contract. The native path currently requires contiguous, +allocated, native-endian float32/float64 or complex64/complex128 vectors; +optional Boolean validity vectors must also be contiguous. Compressed, +chunked, virtual, externally stored, and integer inputs are rejected rather +than delegated to a Python engine. Each signal retains its required +`sample_rate_hz` and `origin_seconds` attributes. + +FP32/FP64 selection, overlapping or separated windows, pair ordering, +linear/circular lags, validity and exposure counts, and result sharding +follow the existing series API. Incomplete result groups are dropped. +The executor captures the CUDA device selected at construction; separate +processes can select GPUs with `CUDA_VISIBLE_DEVICES` and partition result +ordinals using `shard_count` and `shard_index` in `correlation`. + +Reads use two reusable slots; `io_queue_depth` must currently be 2. +`read_buffer_bytes` bounds each raw input span, reducing the processing batch +when needed. A single window must fit that bound. `prefetch_batches=0` +disables the overlap. Results are host-owned; no borrowed device arrays +are exposed by this backend. diff --git a/packaging/conda/recipe.yaml b/packaging/conda/recipe.yaml index 1c4513e6..b606bcf6 100644 --- a/packaging/conda/recipe.yaml +++ b/packaging/conda/recipe.yaml @@ -29,6 +29,7 @@ requirements: - setuptools >=84.2.1 - setuptools-scm ==10.3.4 - wheel + - cython >=3.1,<4 - pybind11 ==3.0.4 - cuda-version ==13.0 - cuda-cudart-dev diff --git a/pyproject.toml b/pyproject.toml index f6509469..26549eb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 [build-system] -requires = ["setuptools>=84.2.1", "setuptools-scm==10.3.4", "wheel"] +requires = ["setuptools>=84.2.1", "setuptools-scm==10.3.4", "wheel", "Cython>=3.1,<4"] build-backend = "setuptools.build_meta" [project] @@ -18,6 +18,7 @@ license-files = [ "THIRD_PARTY_NOTICES.md", "licenses/CCCL-3.1.2.txt", "licenses/CFITSIO-4.7.0-Bison.txt", + "src/cuphoton/xbeam/native/third_party/nlohmann/LICENSE.MIT", ] authors = [{ name = "NVIDIA Corporation" }] keywords = ["astronomy", "cuda", "gpu", "image-processing"] @@ -132,6 +133,7 @@ native-build = [ "setuptools>=84.2.1", "setuptools-scm==10.3.4", "wheel", + "Cython>=3.1,<4", "pybind11>=3.0,<4", ] notebook = [ @@ -163,6 +165,7 @@ where = ["src"] include = ["cuphoton*"] [tool.setuptools.package-data] +"cuphoton.xbeam.native" = ["*.pyi"] "cuphoton.xbeam" = ["schemas/*.json"] "cuphoton.xlens" = ["assets/*.html", "assets/*.css", "assets/*.js"] "cuphoton.xpois" = ["assets/*"] diff --git a/scripts/wheels/build-requirements.txt b/scripts/wheels/build-requirements.txt index 2858e173..6277312c 100644 --- a/scripts/wheels/build-requirements.txt +++ b/scripts/wheels/build-requirements.txt @@ -7,6 +7,7 @@ setuptools==84.2.1 setuptools-scm==10.3.4 vcs-versioning==2.5.0 wheel==0.48.0 +Cython==3.3.1 pybind11==3.0.4 libkvikio-cu13==26.6.0 rapids-logger==0.2.3 diff --git a/scripts/wheels/check_distributions.py b/scripts/wheels/check_distributions.py index d5f40ec5..2b098863 100755 --- a/scripts/wheels/check_distributions.py +++ b/scripts/wheels/check_distributions.py @@ -75,6 +75,12 @@ "src/cuphoton/xray/src/structured_roots.cpp", "src/cuphoton/xray/src/structured_refine.cpp", "src/cuphoton/xray/src/structured_roots.cu", + "tests/xbeam/native/header_smoke.cpp", + "tests/xbeam/native/index_selection_smoke.cpp", + "tests/xbeam/native/radio_options_smoke.cpp", + "tests/xbeam/native/request_smoke.cpp", + "tests/xbeam/native/work_queue_smoke.cpp", + "tests/xbeam/native/run_report_smoke.cpp", "scripts/wheels/prepare_cfitsio.sh", "scripts/wheels/patches/cfitsio-version-string.patch", "scripts/wheels/install_build_dependencies.sh", diff --git a/setup.py b/setup.py index 8fea4dda..b08ee099 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,8 @@ def _build_enabled(variable, *, default=False): extensions = [] commands = {} +if _build_enabled("CUPHOTON_XBEAM_BUILD_EXT"): + extensions.extend(_load_build_helpers("xbeam").get_extensions()) if _build_enabled("CUPHOTON_XDR_BUILD_EXT"): extensions.extend(_load_build_helpers("xdr").get_extensions()) if _build_enabled("CUPHOTON_XRAY_BUILD_EXT"): diff --git a/src/cuphoton/xbeam/commands.py b/src/cuphoton/xbeam/commands.py index c4a56c02..6eaa73f0 100644 --- a/src/cuphoton/xbeam/commands.py +++ b/src/cuphoton/xbeam/commands.py @@ -176,6 +176,12 @@ class BackendInvariant(SetInvariant): _set = {"cupy", "numpy"} +class SeriesBackendInvariant(BackendInvariant): + """Series execution backends, including the optional native CUDA path.""" + + _set = BackendInvariant._set | {"native"} + + class VdifIoInvariant(SetInvariant): """Supported file-to-device VDIF data paths.""" @@ -1039,7 +1045,7 @@ class OutputPathArg(PathValueInvariant): _mandatory = True _metavar = "OUTPUT" - class BackendArg(BackendInvariant): + class BackendArg(SeriesBackendInvariant): _arg = "--backend" _help = "Array backend. [default: %default]" _mandatory = False @@ -1529,10 +1535,123 @@ def _write_manifest(output: Path, manifest: dict[str, object]) -> None: ) +class NativeCorrelateCommand(XBeamCommand): + """Run the experimental native radio engine through Cython.""" + + job: str + output_path: Path + index_dir: Path + scan: str + integration_count: int + profile: str + first_integration = 0 + fft_tile = 1250 + device = 0 + vdif_io = "posix" + upload_workers = 4 + prefetch = "on" + + class JobArg(ExistingPathInvariant): + _help = "Supported .input job; loaded entirely by the native engine." + _metavar = "JOB" + + class OutputPathArg(PathValueInvariant): + _arg = "--output" + _help = "New native SWIN output directory." + _mandatory = True + + class IndexDirArg(ExistingPathInvariant): + _arg = "--index-dir" + _help = "Directory containing validated VDIF sidecars." + _mandatory = True + + class ScanArg(StringInvariant): + _arg = "--scan" + _help = "Scan identifier." + _mandatory = True + + class IntegrationCountArg(PositiveIntegerInvariant): + _arg = "--integration-count" + _help = "Number of complete integrations." + _mandatory = True + + class ProfileArg(StringInvariant): + _arg = "--profile" + _help = "Explicit native processing contract." + _mandatory = True + + class FirstIntegrationArg(NonNegativeIntegerInvariant): + _arg = "--first-integration" + _help = "Zero-based first integration." + _mandatory = False + _default = 0 + + class FftTileArg(PositiveIntegerInvariant): + _arg = "--fft-tile" + _help = "Maximum FFT rows per native processing tile." + _mandatory = False + _default = 1250 + + class DeviceArg(NonNegativeIntegerInvariant): + _arg = "--device" + _help = "Logical CUDA device." + _mandatory = False + _default = 0 + + class VdifIoArg(VdifIoInvariant): + _arg = "--vdif-io" + _help = "Native input delivery: posix or cufile." + _mandatory = False + _default = "posix" + + class UploadWorkersArg(PositiveIntegerInvariant): + _arg = "--upload-workers" + _help = "Maximum native read workers." + _mandatory = False + _default = 4 + + class PrefetchArg(SetInvariant): + _arg = "--prefetch" + _help = "Overlap delivery with GPU processing: on or off." + _set = {"on", "off"} + _mandatory = False + _default = "on" + + def run(self) -> None: + self._call(self._run_native) + + def _run_native(self) -> None: + from .native import correlate, cuda_available + + if not cuda_available(): + raise RuntimeError( + "native CUDA execution was not built; enable " + "CUPHOTON_XBEAM_BUILD_EXT=1 and CUPHOTON_XBEAM_NATIVE_CUDA=1 " + "during installation" + ) + self._emit_json( + correlate( + str(Path(self.job).expanduser()), + self.output_path.expanduser(), + Path(self.index_dir).expanduser(), + self.scan, + first_integration=self.first_integration, + integration_count=self.integration_count, + fft_tile=self.fft_tile, + device=self.device, + vdif_io=self.vdif_io, + profile=self.profile, + upload_workers=self.upload_workers, + prefetch=self.prefetch == "on", + ) + ) + + __all__ = [ "BuildVdifSidecarCommand", "CorrelateCommand", "CorrelateSeriesCommand", "InspectVdifCommand", "MergeShardsCommand", + "NativeCorrelateCommand", ] diff --git a/src/cuphoton/xbeam/core/series.py b/src/cuphoton/xbeam/core/series.py index caf29a0b..2351dc5a 100644 --- a/src/cuphoton/xbeam/core/series.py +++ b/src/cuphoton/xbeam/core/series.py @@ -317,6 +317,11 @@ def correlate_series( Device results expose wait(stream) and to_host() for explicit downstream ownership. Changing a consumer stream cannot race input-slot reuse. """ + if spec.backend == "native": + from ..native.series import correlate_source + + yield from correlate_source(source, spec) + return xp = get_backend(spec.backend) if spec.backend != "cupy": yield from _correlate_series(source, spec) diff --git a/src/cuphoton/xbeam/native/CMakeLists.txt b/src/cuphoton/xbeam/native/CMakeLists.txt new file mode 100644 index 00000000..2c61a7b3 --- /dev/null +++ b/src/cuphoton/xbeam/native/CMakeLists.txt @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +cmake_minimum_required(VERSION 3.24) +project(xbeam_native LANGUAGES CXX) +add_library(xbeam_radio STATIC src/vdif.cpp src/swin.cpp src/requests.cpp src/job_text.cpp src/job.cpp src/index.cpp src/timeline.cpp src/replay.cpp src/window.cpp src/recording.cpp src/execution_bridge.cpp) +target_include_directories(xbeam_radio PUBLIC include PRIVATE third_party) +target_compile_features(xbeam_radio PUBLIC cxx_std_17) +set_target_properties(xbeam_radio PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_compile_options(xbeam_radio PRIVATE -Wall -Wextra -Werror -ffp-contract=off) +find_package(OpenSSL REQUIRED COMPONENTS Crypto) +target_link_libraries(xbeam_radio PUBLIC OpenSSL::Crypto) +add_executable(xbeam-native src/main.cpp) +target_link_libraries(xbeam-native PRIVATE xbeam_radio) +install(TARGETS xbeam_radio xbeam-native) +install(DIRECTORY include/ DESTINATION include) + +option(XBEAM_ENABLE_CUDA "Build native CUDA execution" OFF) +if(XBEAM_ENABLE_CUDA) + enable_language(CUDA) + find_package(CUDAToolkit REQUIRED) + if(CUDAToolkit_VERSION VERSION_LESS 13 OR NOT CUDAToolkit_VERSION VERSION_LESS 14) + message(FATAL_ERROR "The native xBeam backend requires a CUDA 13 toolkit") + endif() + find_package(Python3 REQUIRED COMPONENTS Interpreter) + set(generated "${CMAKE_CURRENT_BINARY_DIR}/kernels") + add_custom_command(OUTPUT "${generated}/front.inc" "${generated}/channel.inc" "${generated}/products.inc" "${generated}/multi_products.inc" + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/cuda/generate.py" + "${CMAKE_CURRENT_SOURCE_DIR}/.." "${generated}" + DEPENDS cuda/generate.py ../cuda_front.py ../cuda_channelize.py ../core/cuda_products.py) + add_library(xbeam_cuda STATIC cuda/front.cu cuda/channel.cu cuda/products.cu cuda/multi_products.cu cuda/station.cpp cuda/input.cu cuda/input.cpp cuda/series.cpp cuda/series_kernels.cu + "${generated}/front.inc" "${generated}/channel.inc" "${generated}/products.inc" "${generated}/multi_products.inc") + target_include_directories(xbeam_cuda PUBLIC include PRIVATE "${generated}" third_party) + target_compile_features(xbeam_cuda PUBLIC cxx_std_17) + set_target_properties(xbeam_cuda PROPERTIES CUDA_STANDARD 17 POSITION_INDEPENDENT_CODE ON) + target_link_libraries(xbeam_cuda PUBLIC xbeam_radio CUDA::cudart CUDA::cufft) +endif() +if(XBEAM_ENABLE_CUDA) + add_executable(xbeam-native-station-test cuda/station_test.cpp) + target_link_libraries(xbeam-native-station-test PRIVATE xbeam_cuda) +endif() +if(XBEAM_ENABLE_CUDA) + add_executable(xbeam-native-station-file-test cuda/station_file_test.cpp) + target_include_directories(xbeam-native-station-file-test PRIVATE third_party) + target_link_libraries(xbeam-native-station-file-test PRIVATE xbeam_cuda) +endif() +if(XBEAM_ENABLE_CUDA) + add_executable(xbeam-native-products-test cuda/products_test.cpp) + target_link_libraries(xbeam-native-products-test PRIVATE xbeam_cuda) +endif() +if(XBEAM_ENABLE_CUDA) + add_library(xbeam_execution STATIC cuda/radio_run.cpp cuda/radio_options.cpp cuda/radio_model.cpp) + target_include_directories(xbeam_execution PRIVATE third_party) + target_compile_options(xbeam_execution PRIVATE -ffp-contract=off) + set_target_properties(xbeam_execution PROPERTIES POSITION_INDEPENDENT_CODE ON) + find_package(Threads REQUIRED) + target_link_libraries(xbeam_execution PUBLIC xbeam_cuda Threads::Threads) + add_executable(xbeam-native-correlate cuda/main.cpp) + target_link_libraries(xbeam-native-correlate PRIVATE xbeam_execution) + install(TARGETS xbeam_execution xbeam_cuda xbeam-native-correlate) +endif() + +option(XBEAM_ENABLE_CUFILE "Enable native cuFile reads" OFF) +if(XBEAM_ENABLE_CUDA AND XBEAM_ENABLE_CUFILE) + find_library(XBEAM_CUFILE_LIBRARY cufile HINTS "${CUDAToolkit_LIBRARY_DIR}" "${CUDAToolkit_ROOT}/lib64" REQUIRED) + target_compile_definitions(xbeam_cuda PRIVATE XBEAM_HAVE_CUFILE=1) + target_link_libraries(xbeam_cuda PUBLIC "${XBEAM_CUFILE_LIBRARY}") +endif() +if(XBEAM_ENABLE_CUDA) + add_executable(xbeam-native-input-test cuda/input_test.cpp) + target_link_libraries(xbeam-native-input-test PRIVATE xbeam_cuda Threads::Threads) +endif() diff --git a/src/cuphoton/xbeam/native/__init__.py b/src/cuphoton/xbeam/native/__init__.py new file mode 100644 index 00000000..c4d121e5 --- /dev/null +++ b/src/cuphoton/xbeam/native/__init__.py @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Optional native CUDA execution with a thin Cython interface.""" + + +def cuda_available() -> bool: + """Whether the optional extension includes native CUDA execution.""" + try: + from ._binding import cuda_available as available + except ModuleNotFoundError as error: + if error.name != __name__ + "._binding": + raise + return False + return available() + + +def correlate(*args, **kwargs): + """Run native radio IO, planning, CUDA execution, and SWIN writing. + + The optional extension releases the GIL for the complete C++ run. It + takes file paths and a bounded scan/integration scope; no Python job or + GPU-array objects participate in native execution. + """ + from ._binding import correlate as execute + + return execute(*args, **kwargs) diff --git a/src/cuphoton/xbeam/native/_binding.pyi b/src/cuphoton/xbeam/native/_binding.pyi new file mode 100644 index 00000000..a54f033c --- /dev/null +++ b/src/cuphoton/xbeam/native/_binding.pyi @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Typed execution entry points for the optional Cython extension.""" + +from os import PathLike +from typing import Any + +def cuda_available() -> bool: ... +def correlate( + job: str | PathLike[str], + output: str | PathLike[str], + index_dir: str | PathLike[str], + scan: str, + first_integration: int = ..., + fft_tile: int = ..., + device: int = ..., + vdif_io: str = ..., + *, + integration_count: int, + profile: str, + upload_workers: int = ..., + prefetch: bool = ..., +) -> dict[str, Any]: ... diff --git a/src/cuphoton/xbeam/native/_binding.pyx b/src/cuphoton/xbeam/native/_binding.pyx new file mode 100644 index 00000000..95d7932d --- /dev/null +++ b/src/cuphoton/xbeam/native/_binding.pyx @@ -0,0 +1,550 @@ +# SPDX-License-Identifier: Apache-2.0 +# cython: language_level=3 +from libc.stddef cimport size_t +from libc.stdint cimport uint32_t, uint64_t +from libcpp.string cimport string +cdef extern from "xbeam/radio.hpp" namespace "xbeam": + cdef cppclass VdifHeader: + bint invalid, complex_data + uint32_t seconds_from_epoch, reference_epoch, frame_number + uint32_t version, log2_channels, frame_length_bytes + uint32_t bits_per_sample, thread_id, station_id, extended_data_version + uint64_t samples_per_frame + VdifHeader parse_vdif_header(const unsigned char*, size_t, bint, bint) except + nogil + string inspect_vdif_json(const string&) except + nogil + +def parse_header(const unsigned char[::1] data, bint strict=True, bint require_two_bit=True): + cdef VdifHeader h + cdef size_t n=data.shape[0] + cdef const unsigned char* pointer=NULL + if n: + pointer=&data[0] + with nogil: + h=parse_vdif_header(pointer,n,strict,require_two_bit) + return dict(invalid=h.invalid, complex_data=h.complex_data, + seconds_from_epoch=h.seconds_from_epoch, reference_epoch=h.reference_epoch, + frame_number=h.frame_number, version=h.version, log2_channels=h.log2_channels, + frame_length_bytes=h.frame_length_bytes, bits_per_sample=h.bits_per_sample, + thread_id=h.thread_id, station_id=h.station_id, + extended_data_version=h.extended_data_version, samples_per_frame=h.samples_per_frame) + +def inspect_recording(path): + import json + import os + cdef string encoded=os.fsencode(path) + cdef string result + with nogil: + result=inspect_vdif_json(encoded) + return json.loads(result.decode('utf-8')) + +cdef extern from "xbeam/radio.hpp" namespace "xbeam": + cdef cppclass SwinMetadata: + int baseline, configuration_id, source_id, frequency_id + unsigned int mjd, pulsar_bin + double seconds, weight, u, v, w + char polarization_a, polarization_b + string encode_swin(const SwinMetadata&, const float*, size_t) except + nogil + void write_swin_file(const string&, const string&) except + nogil + +def encode_visibility(int baseline, unsigned int mjd, double seconds, + int configuration_id, int source_id, int frequency_id, + bytes polarization, unsigned int pulsar_bin, + double weight, double u, double v, double w, + const float[::1] interleaved): + """Encode an observable SWIN record using the native codec.""" + if len(polarization)!=2: + raise ValueError("polarization requires two ASCII bytes") + if interleaved.shape[0]%2: + raise ValueError("spectrum requires interleaved complex pairs") + cdef SwinMetadata m + m.baseline=baseline; m.mjd=mjd; m.seconds=seconds + m.configuration_id=configuration_id; m.source_id=source_id + m.frequency_id=frequency_id; m.pulsar_bin=pulsar_bin + m.polarization_a=polarization[0]; m.polarization_b=polarization[1] + m.weight=weight; m.u=u; m.v=v; m.w=w + cdef const float* pointer=NULL + cdef size_t n=interleaved.shape[0]//2 + if n: + pointer=&interleaved[0] + cdef string result + with nogil: + result=encode_swin(m,pointer,n) + return result + +def write_visibility_bytes(path, bytes records): + """Create a new SWIN stream; existing paths are never overwritten.""" + import os + cdef string name=os.fsencode(path) + cdef string payload=records + with nogil: + write_swin_file(name,payload) + +cdef extern from "xbeam/radio.hpp" namespace "xbeam": + cdef cppclass RequestGeometry: + long long ratio_numerator, ratio_denominator, fft_samples + long long ffts_per_request, alignment_bytes, word_bytes + cdef cppclass RequestCursor: + bint valid + long long origin_bytes, skipped_ffts, decoder_delta_samples + RequestCursor request_cursor(long long, double, const RequestGeometry&, double) except + nogil + +def buffered_cursor(long long local_samples, double drift_seconds, + long long ratio_numerator, long long ratio_denominator, + long long fft_samples, long long ffts_per_request, + long long alignment_bytes, long long word_bytes, + double sample_rate): + cdef RequestGeometry g + g.ratio_numerator=ratio_numerator; g.ratio_denominator=ratio_denominator + g.fft_samples=fft_samples; g.ffts_per_request=ffts_per_request + g.alignment_bytes=alignment_bytes; g.word_bytes=word_bytes + cdef RequestCursor result + with nogil: + result=request_cursor(local_samples,drift_seconds,g,sample_rate) + if not result.valid: + return None + return (result.origin_bytes,result.skipped_ffts,result.decoder_delta_samples) + +from libcpp.vector cimport vector +cdef extern from "xbeam/radio.hpp" namespace "xbeam": + cdef cppclass TextEntry: + string key, value + size_t line_number + vector[TextEntry] parse_job_text(const string&) except + nogil + +def parse_text(str text): + cdef string encoded=text.encode('utf-8') + cdef vector[TextEntry] entries + cdef size_t i + with nogil: + entries=parse_job_text(encoded) + return [(entries[i].key.decode('utf-8'),entries[i].value.decode('utf-8'), + entries[i].line_number) for i in range(entries.size())] + +cdef extern from "xbeam/radio.hpp" namespace "xbeam": + string load_interop_job_json(const string&) except + nogil + +def load_text_job(path): + import json + import os + cdef string name=os.fsencode(path) + cdef string result + with nogil: + result=load_interop_job_json(name) + return json.loads(result.decode('utf-8')) + +cdef extern from "xbeam/radio.hpp" namespace "xbeam": + cdef cppclass IndexFrame: + unsigned long long timestamp, physical_frame + unsigned short thread + unsigned char state + cdef cppclass PhysicalIndexRun: + uint64_t first,stop,packed_timestamp + cdef cppclass IndexReader: + IndexReader(const string&, const string&) except + nogil + string metadata_json() except + nogil + vector[PhysicalIndexRun] physical_runs() except + nogil + vector[unsigned char] frame_validity(uint64_t,uint64_t) except + nogil + vector[IndexFrame] query(unsigned short, unsigned long long, unsigned long long) except + nogil + +cdef class NativeIndex: + cdef IndexReader* reader + def __cinit__(self, source, index): + import os + cdef string a=os.fsencode(source) + cdef string b=os.fsencode(index) + self.reader=NULL + with nogil: + self.reader=new IndexReader(a,b) + def __dealloc__(self): + del self.reader + def metadata(self): + import json + cdef string result + with nogil: + result=self.reader.metadata_json() + return json.loads(result.decode('utf-8')) + def physical_runs(self): + cdef vector[PhysicalIndexRun] runs + with nogil: + runs=self.reader.physical_runs() + return [(r.first,r.stop,r.packed_timestamp) for r in runs] + def frame_validity(self,uint64_t first,uint64_t count): + cdef vector[unsigned char] result + with nogil: + result=self.reader.frame_validity(first,count) + return list(result) + def query(self, unsigned short thread, unsigned long long first, unsigned long long stop): + cdef vector[IndexFrame] rows + cdef size_t i + with nogil: + rows=self.reader.query(thread,first,stop) + return [(rows[i].timestamp,rows[i].physical_frame,rows[i].thread,rows[i].state) + for i in range(rows.size())] + +cdef extern from "xbeam/radio.hpp" namespace "xbeam": + cdef cppclass SourceBlock: + long long fft_samples, output_sample, coarse_sample + double centre_delay + cdef cppclass RadioJob: + RadioJob(const string&) except + nogil + string normalized_json() except + nogil + long long frame_slot(long long,int) except + nogil + SwinMetadata record_metadata(const string&,long long,int,int,int,char,char) except + nogil + vector[SourceBlock] source_blocks(const string&,int,int,const double*,size_t,bint) except + nogil + JobRequestInputs requests(const string&,int,int,const RequestGeometry&,long long,long long,long long,long long,long long,bint,long long) except + nogil + +cdef class NativeJob: + cdef RadioJob* job + def __cinit__(self,path): + import os + cdef string name=os.fsencode(path) + self.job=NULL + with nogil: + self.job=new RadioJob(name) + def __dealloc__(self): + del self.job + def record_metadata(self,str scan,long long integration,int station_a,int station_b,int frequency,bytes polarization): + if len(polarization)!=2: + raise ValueError("polarization requires two ASCII bytes") + cdef string name=scan.encode('utf-8') + cdef char pa=polarization[0],pb=polarization[1] + cdef SwinMetadata m + with nogil: + m=self.job.record_metadata(name,integration,station_a,station_b,frequency,pa,pb) + return dict(baseline=m.baseline,mjd=m.mjd,seconds=m.seconds, + configuration_id=m.configuration_id,source_id=m.source_id, + frequency_id=m.frequency_id,uvw=(m.u,m.v,m.w)) + def frame_slot(self,long long fps,int epoch): + return self.job.frame_slot(fps,epoch) + def source_blocks(self,str scan,int stream,int band,const double[::1] starts): + cdef string name=scan.encode('utf-8') + cdef vector[SourceBlock] result + cdef const double* pointer=NULL + cdef size_t n=starts.shape[0],i + if n: + pointer=&starts[0] + with nogil: + result=self.job.source_blocks(name,stream,band,pointer,n,False) + return [(result[i].fft_samples,result[i].output_sample, + result[i].coarse_sample,result[i].centre_delay) for i in range(result.size())] + + def requests(self,str scan,int stream,int band,long long num,long long den, + long long fft,long long width,long long alignment,long long word, + long long first,long long stop,long long timeline_count, + long long job_slot,long long timestamp_origin,bint endpoint_mean=True, + long long maximum_requests=4096): + cdef string name=scan.encode('utf-8') + cdef RequestGeometry g + g.ratio_numerator=num;g.ratio_denominator=den;g.fft_samples=fft + g.ffts_per_request=width;g.alignment_bytes=alignment;g.word_bytes=word + cdef JobRequestInputs result + cdef size_t i + with nogil: + result=self.job.requests(name,stream,band,g,first,stop,timeline_count, + job_slot,timestamp_origin,endpoint_mean,maximum_requests) + return dict(first_fft=result.first_fft,stop_fft=result.stop_fft, + timeline_fft_count=result.timeline_fft_count, + timestamp_origin_frame=result.timestamp_origin,sample_rate=result.sample_rate, + request_fft_starts=[r.first_fft for r in result.requests], + request_fft_counts=[r.active_count for r in result.requests], + desired_sample_starts=[r.desired_sample for r in result.requests], + endpoint_drifts=[r.endpoint_drift for r in result.requests], + ordinary_source_starts=[list(r.ordinary_samples) for r in result.requests], + output_sample_starts=[list(r) for r in result.output_samples], + centre_delays_seconds=[list(r) for r in result.centre_delays], + read_average_delays_seconds=[list(r) for r in result.read_average], + left_delays=list(result.left_delays)) + +from libc.stdint cimport int64_t +cdef extern from "xbeam/timeline.hpp" namespace "xbeam": + cdef cppclass PhysicalRun: + int64_t first, stop, timestamp + cdef cppclass SampleSpan: + int64_t request_first, request_stop, physical_first + cdef cppclass AnchorGap: + int64_t allocation, physical_boundary, first, stop + cdef cppclass FrameInterval: + int64_t first,stop + cdef cppclass PhysicalTimeline: + PhysicalTimeline(const vector[PhysicalRun]&,int64_t) except + nogil + const vector[int64_t]& allocation_starts() noexcept nogil + const vector[int64_t]& anchors() noexcept nogil + vector[FrameInterval] uncovered_heads() except + nogil + vector[AnchorGap] deferred_gaps() except + nogil + vector[SampleSpan] source_spans(int64_t,int64_t,int64_t) except + nogil + vector[SampleSpan] latched_spans(int64_t,int64_t,int64_t,int64_t) except + nogil + +cdef class NativeTimeline: + cdef PhysicalTimeline* timeline + def __cinit__(self, runs, int64_t allocation_frames): + cdef vector[PhysicalRun] values + cdef PhysicalRun item + self.timeline=NULL + for first,stop,timestamp in runs: + item.first=first;item.stop=stop;item.timestamp=timestamp + values.push_back(item) + with nogil: + self.timeline=new PhysicalTimeline(values,allocation_frames) + def __dealloc__(self): + del self.timeline + def snapshot(self): + cdef vector[int64_t] first=self.timeline.allocation_starts() + cdef vector[int64_t] anchors=self.timeline.anchors() + cdef vector[FrameInterval] heads + cdef vector[AnchorGap] gaps + cdef size_t i + with nogil: + heads=self.timeline.uncovered_heads() + gaps=self.timeline.deferred_gaps() + return dict(allocation_starts=list(first),anchors=list(anchors), + uncovered_heads=[(heads[i].first,heads[i].stop) for i in range(heads.size())], + deferred_gaps=[(gaps[i].allocation,gaps[i].physical_boundary,gaps[i].first,gaps[i].stop) for i in range(gaps.size())]) + def source_spans(self,int64_t first,int64_t count,int64_t samples_per_frame): + cdef vector[SampleSpan] result + cdef size_t i + with nogil: + result=self.timeline.source_spans(first,count,samples_per_frame) + return [(result[i].request_first,result[i].request_stop,result[i].physical_first) for i in range(result.size())] + def latched_spans(self,int64_t allocation,int64_t first,int64_t count,int64_t samples_per_frame): + cdef vector[SampleSpan] result + cdef size_t i + with nogil: + result=self.timeline.latched_spans(allocation,first,count,samples_per_frame) + return [(result[i].request_first,result[i].request_stop,result[i].physical_first) for i in range(result.size())] + +cdef extern from "xbeam/replay.hpp" namespace "xbeam": + cdef cppclass RawRequestState: + RawRequestState() except + + vector[int64_t] anchors + vector[unsigned char] events + cdef cppclass RequestFact: + int64_t first_fft,desired_sample,active_count + double endpoint_drift + vector[int64_t] ordinary_samples + cdef cppclass JobRequestInputs: + long long first_fft,stop_fft,timeline_fft_count,timestamp_origin + double sample_rate + vector[RequestFact] requests + vector[vector[int64_t]] output_samples + vector[vector[double]] centre_delays,read_average + vector[double] left_delays + cdef cppclass DiscardBatch: + vector[int64_t] allocations + vector[RequestCursor] cursors + vector[unsigned char] valid,events + vector[vector[unsigned char]] eligible + vector[FrameInterval] excluded + RawRequestState raw_request_state(const PhysicalTimeline&,int64_t,int64_t,bint) except + nogil + DiscardBatch compile_discards(const RawRequestState&,const vector[RequestFact]&,const RequestGeometry&,double,int64_t,int64_t) except + nogil + cdef cppclass WindowBounds: + int64_t nearest,valid_first,valid_stop + bint before,after + WindowBounds window_bounds(int64_t,int64_t,double,int64_t,int64_t,int64_t,int64_t) except + nogil + +cdef class NativeRawState: + cdef RawRequestState state + def __cinit__(self,NativeTimeline timeline not None,int64_t samples_per_frame,int64_t timestamp_origin,bint clock_frame_owner): + with nogil: + self.state=raw_request_state(timeline.timeline[0],samples_per_frame,timestamp_origin,clock_frame_owner) + def snapshot(self): + return dict(anchors=list(self.state.anchors),events=list(self.state.events)) + def discards(self,const int64_t[::1] first,const int64_t[::1] desired, + const double[::1] drifts,const int64_t[:,::1] ordinary, + const int64_t[::1] counts, + int64_t num,int64_t den,int64_t fft,int64_t ffts, + int64_t alignment,int64_t word,double rate, + int64_t frame_bytes,int64_t control_bits): + cdef size_t n=first.shape[0],i,j + if desired.shape[0]!=n or drifts.shape[0]!=n or ordinary.shape[0]!=n or counts.shape[0]!=n: + raise ValueError("request vectors must have equal row counts") + cdef vector[RequestFact] facts + cdef RequestFact fact + for i in range(n): + if counts[i]<1 or counts[i]>ordinary.shape[1]: + raise ValueError("request count exceeds row width") + fact.first_fft=first[i];fact.desired_sample=desired[i];fact.endpoint_drift=drifts[i] + fact.ordinary_samples.clear() + for j in range(counts[i]): + fact.ordinary_samples.push_back(ordinary[i,j]) + facts.push_back(fact) + cdef RequestGeometry g + g.ratio_numerator=num;g.ratio_denominator=den;g.fft_samples=fft + g.ffts_per_request=ffts;g.alignment_bytes=alignment;g.word_bytes=word + cdef DiscardBatch result + with nogil: + result=compile_discards(self.state,facts,g,rate,frame_bytes,control_bits) + return dict(allocations=list(result.allocations),valid=list(result.valid), + events=list(result.events),eligible=[list(result.eligible[i]) for i in range(n)], + cursors=[(result.cursors[i].origin_bytes,result.cursors[i].skipped_ffts,result.cursors[i].decoder_delta_samples) for i in range(n)], + excluded=[(result.excluded[i].first,result.excluded[i].stop) for i in range(result.excluded.size())]) + +def returned_window_bounds(int64_t output,int64_t head,double delay,int64_t rate, + int64_t framed_extent,int64_t decoded_extent,int64_t fft): + cdef WindowBounds result + with nogil: + result=window_bounds(output,head,delay,rate,framed_extent,decoded_extent,fft) + return (result.nearest,result.before,result.after,result.valid_first,result.valid_stop) + +cdef extern from "xbeam/window.hpp" namespace "xbeam": + int64_t packed_frame_slot(uint64_t,int64_t) except + nogil + cdef cppclass WindowGeometry: + int64_t frames_per_fft,ffts_per_request,buffered_ffts,ring_segments + int64_t allocation_frames,request_frames,read_buffer_ratio + WindowGeometry size_windows(int64_t,int64_t,int64_t,int64_t,int64_t,int64_t,int64_t,int64_t,double) except + nogil + cdef cppclass NormalExtent: + int64_t frame_bytes,payload_bytes,request_frames,read_buffer_ratio + int64_t ratio_num,ratio_den,samples_per_frame + int64_t framed_admission_samples,decoded_samples,allocation_bytes + NormalExtent normal_extent(int64_t,int64_t,int64_t,int64_t,int64_t,int64_t) except + nogil + int64_t leading_head(int64_t,double,int64_t,int64_t,int64_t) except + nogil + cdef cppclass ReturnedWindow: + int64_t fft,output,logical_head,physical_head + double centre_delay + int64_t framed_extent,decoded_extent + bint published,clamp_negative + vector[ReturnedWindow] returned_windows(const PhysicalTimeline&,const RawRequestState&, + const JobRequestInputs&,const RequestGeometry&, + const NormalExtent&,int64_t,int64_t,int64_t) except + nogil + +def window_geometry(int64_t spf,int64_t fft,int64_t rate,int64_t subint_num,int64_t subint_den, + int64_t factor,int64_t segments,int64_t buffered,double allowance): + cdef WindowGeometry g + with nogil: + g=size_windows(spf,fft,rate,subint_num,subint_den,factor,segments,buffered,allowance) + return (g.frames_per_fft,g.ffts_per_request,g.buffered_ffts,g.ring_segments, + g.allocation_frames,g.request_frames,g.read_buffer_ratio) + +def leading_frame_head(int64_t output,double delay,int64_t rate,int64_t spf,int64_t quantum): + return leading_head(output,delay,rate,spf,quantum) + +def returned_window_contract(NativeTimeline timeline not None,NativeRawState state not None,dict inputs, + tuple geometry,tuple extent,int64_t source_stop, + int64_t quantum=1,int64_t control=30): + cdef RequestGeometry g + g.ratio_numerator,g.ratio_denominator,g.fft_samples,g.ffts_per_request,g.alignment_bytes,g.word_bytes=geometry + cdef NormalExtent e=normal_extent(extent[0],extent[1],extent[2],extent[3],extent[4],extent[5]) + cdef JobRequestInputs data + data.first_fft=inputs['first_fft'];data.stop_fft=inputs['stop_fft'] + data.timeline_fft_count=inputs['timeline_fft_count'];data.timestamp_origin=inputs['timestamp_origin_frame'] + data.sample_rate=inputs['sample_rate'] + cdef RequestFact f + cdef size_t i,j + for i in range(len(inputs['request_fft_starts'])): + f.first_fft=inputs['request_fft_starts'][i];f.active_count=inputs['request_fft_counts'][i] + f.desired_sample=inputs['desired_sample_starts'][i];f.endpoint_drift=inputs['endpoint_drifts'][i] + f.ordinary_samples=inputs['ordinary_source_starts'][i] + data.requests.push_back(f) + data.output_samples=inputs['output_sample_starts'];data.centre_delays=inputs['centre_delays_seconds'] + data.left_delays=inputs['left_delays'] + cdef vector[ReturnedWindow] windows + with nogil: + windows=returned_windows(timeline.timeline[0],state.state,data,g,e,control,quantum,source_stop) + return [(w.fft,w.output,w.logical_head,w.physical_head,w.centre_delay, + w.framed_extent,w.decoded_extent,bool(w.published),bool(w.clamp_negative)) for w in windows] + +def packed_timestamp_slot(uint64_t timestamp,int64_t fps): + return packed_frame_slot(timestamp,fps) + +cdef extern from "xbeam/window.hpp" namespace "xbeam": + cdef cppclass ReplayRow: + int64_t fft,physical_start,timing_displacement,valid_first,valid_stop + bint accepted + vector[ReplayRow] compile_replay_rows(const vector[ReturnedWindow]&,int64_t,int64_t,int64_t) except + nogil + +def replay_rows(windows,int64_t fft,int64_t rate,int64_t source_stop): + cdef vector[ReturnedWindow] rows + cdef ReturnedWindow w + for row in windows: + w.fft,w.output,w.logical_head,w.physical_head,w.centre_delay,w.framed_extent,w.decoded_extent,w.published,w.clamp_negative=row + rows.push_back(w) + cdef vector[ReplayRow] result + with nogil: + result=compile_replay_rows(rows,fft,rate,source_stop) + return [(r.fft,r.physical_start,r.timing_displacement,r.valid_first,r.valid_stop,bool(r.accepted)) for r in result] + +cdef extern from "xbeam/execution.hpp" namespace "xbeam": + string correlate_native(const vector[string]&) except + nogil + bint native_cuda_available() noexcept + +def cuda_available(): + return bool(native_cuda_available()) + +def correlate(job,output,index_dir,str scan,long long first_integration=0, + int fft_tile=1250,int device=0,str vdif_io="posix",*, + long long integration_count,str profile,int upload_workers=4,bint prefetch=True): + """Run native radio execution; release the GIL for the complete run.""" + import os + import json + cdef vector[string] arguments + for argument in [b"xbeam-native-correlate",os.fsencode(job),b"--output",os.fsencode(output), + b"--index-dir",os.fsencode(index_dir),b"--scan",scan.encode('utf-8'), + b"--first-integration",str(first_integration).encode('ascii'), + b"--integration-count",str(integration_count).encode('ascii'), + b"--fft-tile",str(fft_tile).encode('ascii'),b"--device",str(device).encode('ascii'), + b"--vdif-io",vdif_io.encode('ascii'),b"--profile",profile.encode('ascii'), + b"--upload-workers",str(upload_workers).encode('ascii'), + b"--prefetch",b"on" if prefetch else b"off"]: + arguments.push_back(argument) + cdef string report + with nogil: + report=correlate_native(arguments) + return json.loads(report.decode('utf-8')) + + +cdef extern from "xbeam/series.hpp" namespace "xbeam": + cdef cppclass NativeSeries: + pass + NativeSeries* create_native_series(const string&) except + nogil + void destroy_native_series(NativeSeries*) noexcept nogil + void run_native_series(NativeSeries*, int64_t, int, void*, size_t, + void*, size_t, int64_t*, size_t, + int64_t*, size_t) except + nogil + +cdef class SeriesExecutor: + """Native descriptor, IO, and DSP owner. Every submitted result is synchronous.""" + cdef NativeSeries* _owner + cdef bint _active + + def __cinit__(self, configuration): + import json + cdef string encoded = json.dumps(configuration).encode("utf-8") + with nogil: + self._owner = create_native_series(encoded) + + def close(self): + cdef NativeSeries* owner + if self._active: + raise RuntimeError("native series executor is active") + owner = self._owner + self._owner = NULL + if owner != NULL: + with nogil: + destroy_native_series(owner) + + def __dealloc__(self): + if self._owner != NULL: + destroy_native_series(self._owner) + + def run(self, int64_t first, int windows, unsigned char[::1] spectrum, + unsigned char[::1] lag, int64_t[::1] statistics, + int64_t[::1] coverage): + cdef void* spectrum_ptr = NULL + cdef void* lag_ptr = NULL + cdef int64_t* stats_ptr = NULL + cdef int64_t* coverage_ptr = NULL + if self._owner == NULL: + raise RuntimeError("native series executor is closed") + if spectrum.shape[0]: spectrum_ptr = &spectrum[0] + if lag.shape[0]: lag_ptr = &lag[0] + if statistics.shape[0]: stats_ptr = &statistics[0] + if coverage.shape[0]: coverage_ptr = &coverage[0] + if self._active: + raise RuntimeError("native series executor is active") + self._active = True + try: + with nogil: + run_native_series(self._owner, first, windows, spectrum_ptr, + spectrum.shape[0], lag_ptr, lag.shape[0], stats_ptr, + statistics.shape[0], coverage_ptr, coverage.shape[0]) + finally: + self._active = False diff --git a/src/cuphoton/xbeam/native/cuda/channel.cu b/src/cuphoton/xbeam/native/cuda/channel.cu new file mode 100644 index 00000000..7a4d1dbd --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/channel.cu @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "channel.inc" +#include "xbeam/cuda.hpp" +namespace xbeam { +void launch_channel( + const StationLayout& g, + const void* transformed, + const void* parameters, + void* output, + cudaStream_t stream) { + const int offset = g.mapping == 0 ? 0 + : g.mapping == 3 ? -(g.fft_samples / 2) + : -g.channels; + const auto* input = static_cast(transformed); + const auto* p = static_cast(parameters); + auto* result = static_cast(output); + const double spacing = g.sample_rate / g.fft_samples; + switch (g.mapping) { + case 0: + channelize_residual_identity<<>>( + input, p, g.fft_samples, g.channels, offset, spacing, result); + break; + case 1: + channelize_residual_real_lsb<<>>( + input, p, g.fft_samples, g.channels, offset, spacing, result); + break; + case 2: + channelize_residual_complex_lsb<<>>( + input, p, g.fft_samples, g.channels, offset, spacing, result); + break; + case 3: + channelize_residual_complex_dsb<<>>( + input, p, g.fft_samples, g.channels, offset, spacing, result); + break; + } +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/front.cu b/src/cuphoton/xbeam/native/cuda/front.cu new file mode 100644 index 00000000..c5c4bce4 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/front.cu @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "front.inc" +#include "xbeam/cuda.hpp" +namespace xbeam { +void launch_front( + const StationLayout& g, + const void* source, + const void* validity, + const void* parameters, + void* rotated, + void* weights, + void* counts, + cudaStream_t stream) { + const bool chunked = !g.complex_input && g.fft_samples == 262144 + && g.rows >= 8 && g.rows <= 1600; + if (chunked) { + front_chunked_packed_real<<>>( + static_cast(source), + static_cast(validity), + g.validity_stride, + g.authoritative_validity, + g.source_samples, + g.samples_per_frame, + g.payload_bytes, + g.channel_count, + g.channel_index, + g.source_start, + static_cast(parameters), + g.fft_samples, + 1. / g.sample_rate, + g.reference_hz, + g.lo_hz, + g.wall_lo_cycles, + static_cast(rotated), + static_cast(counts)); + finish_front_weights<<>>( + static_cast(counts), + 64, + g.fft_samples, + static_cast(weights)); + } else if (g.complex_input) { + front_packed_complex<<>>( + static_cast(source), + static_cast(validity), + g.validity_stride, + g.authoritative_validity, + g.source_samples, + g.samples_per_frame, + g.payload_bytes, + g.channel_count, + g.channel_index, + g.source_start, + static_cast(parameters), + g.fft_samples, + 1. / g.sample_rate, + g.reference_hz, + g.lo_hz, + g.wall_lo_cycles, + static_cast(rotated), + static_cast(weights)); + } else { + front_packed_real<<>>( + static_cast(source), + static_cast(validity), + g.validity_stride, + g.authoritative_validity, + g.source_samples, + g.samples_per_frame, + g.payload_bytes, + g.channel_count, + g.channel_index, + g.source_start, + static_cast(parameters), + g.fft_samples, + 1. / g.sample_rate, + g.reference_hz, + g.lo_hz, + g.wall_lo_cycles, + static_cast(rotated), + static_cast(weights)); + } +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/generate.py b/src/cuphoton/xbeam/native/cuda/generate.py new file mode 100644 index 00000000..b69fb5f5 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/generate.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Extract owned CUDA sources without importing runtime modules.""" + +import ast +import sys +from pathlib import Path + +root, output = map(Path, sys.argv[1:]) +output.mkdir(parents=True, exist_ok=True) +for filename, symbol, destination in [ + ("cuda_front.py", "_FRONT_SOURCE", "front.inc"), + ("cuda_channelize.py", "_CHANNELIZE_RESIDUAL_SOURCE", "channel.inc"), + ("core/cuda_products.py", "_VISIBILITY_SOURCE", "products.inc"), + ( + "core/cuda_products.py", + "_MULTI_VISIBILITY_SOURCE", + "multi_products.inc", + ), +]: + tree = ast.parse((root / filename).read_text()) + values = [ + ast.literal_eval(node.value) + for node in tree.body + if isinstance(node, ast.Assign) + and any( + isinstance(t, ast.Name) and t.id == symbol for t in node.targets + ) + ] + if len(values) != 1: + raise RuntimeError(f"expected one {symbol} definition") + (output / destination).write_text( + "// SPDX-License-Identifier: Apache-2.0\n" + values[0] + ) diff --git a/src/cuphoton/xbeam/native/cuda/gpu.hpp b/src/cuphoton/xbeam/native/cuda/gpu.hpp new file mode 100644 index 00000000..27e1f404 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/gpu.hpp @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 +// Small RAII wrappers over the CUDA runtime, shared by the native execution code. +#pragma once +#include "xbeam/radio.hpp" +#include +#include +#include +#include +namespace xbeam::gpu { + +inline void check(cudaError_t status) { + if (status != cudaSuccess) { + throw RadioError(cudaGetErrorString(status)); + } +} + +// Grow-only device buffer. `alignment` > 1 over-allocates so `p` is aligned +// (NVFS needs 64-KiB aligned device pages); the owning pointer is retained. +struct Memory { + void* p = nullptr; + std::size_t capacity = 0; + explicit Memory(std::size_t alignment = 1) + : alignment_(alignment) {} + Memory(const Memory&) = delete; + Memory& operator=(const Memory&) = delete; + ~Memory() { + if (allocation_) { + cudaFree(allocation_); + } + } + void reserve(std::size_t bytes) { + bytes = (bytes + alignment_ - 1) / alignment_ * alignment_; + if (bytes <= capacity) { + return; + } + if (allocation_) { + check(cudaFree(allocation_)); + allocation_ = p = nullptr; + capacity = 0; + } + check(cudaMalloc(&allocation_, bytes + alignment_ - 1)); + p = reinterpret_cast( + (reinterpret_cast(allocation_) + alignment_ - 1) + / alignment_ * alignment_); + capacity = bytes; + } + template + void upload(const std::vector& data) { + reserve(data.size() * sizeof(T)); + check(cudaMemcpy( + p, data.data(), data.size() * sizeof(T), cudaMemcpyHostToDevice)); + } + std::uintptr_t address() const { + return reinterpret_cast(p); + } + +private: + std::size_t alignment_; + void* allocation_ = nullptr; +}; + +// Grow-only page-locked host buffer. +struct Pinned { + void* p = nullptr; + std::size_t capacity = 0; + Pinned() = default; + Pinned(const Pinned&) = delete; + Pinned& operator=(const Pinned&) = delete; + ~Pinned() { + if (p) { + cudaFreeHost(p); + } + } + void reserve(std::size_t bytes) { + if (bytes <= capacity) { + return; + } + if (p) { + check(cudaFreeHost(p)); + p = nullptr; + capacity = 0; + } + check(cudaHostAlloc(&p, bytes, cudaHostAllocDefault)); + capacity = bytes; + } + template + T* as() const { + return static_cast(p); + } +}; + +struct Stream { + cudaStream_t value = nullptr; + Stream() { + check(cudaStreamCreateWithFlags(&value, cudaStreamNonBlocking)); + } + Stream(const Stream&) = delete; + Stream& operator=(const Stream&) = delete; + ~Stream() { + if (value) { + cudaStreamSynchronize(value); + cudaStreamDestroy(value); + } + } +}; + +struct Event { + cudaEvent_t value = nullptr; + Event() { + check(cudaEventCreateWithFlags(&value, cudaEventDisableTiming)); + } + Event(const Event&) = delete; + Event& operator=(const Event&) = delete; + ~Event() { + if (value) { + cudaEventDestroy(value); + } + } +}; + +} // namespace xbeam::gpu diff --git a/src/cuphoton/xbeam/native/cuda/input.cpp b/src/cuphoton/xbeam/native/cuda/input.cpp new file mode 100644 index 00000000..15f2d0ca --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/input.cpp @@ -0,0 +1,448 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "gpu.hpp" +#include "xbeam/cuda.hpp" +#include "xbeam/series.hpp" +#include "xbeam/radio.hpp" +#include +#ifdef XBEAM_HAVE_CUFILE +#include +#endif +#include +#include +#include +#include +#include +#include +namespace xbeam { +namespace { +using gpu::check; +// NVFS and cuFile require 64-KiB aligned device pages. +constexpr std::size_t kDevicePage = 65536; +#ifdef XBEAM_HAVE_CUFILE +struct Driver { + Driver() { + auto result = cuFileDriverOpen(); + if (result.err != CU_FILE_SUCCESS) { + throw RadioError( + "cuFile driver open failed: " + std::to_string(result.err)); + } + } + ~Driver() { + cuFileDriverClose(); + } +}; +struct ThreadStaging { + gpu::Memory raw{kDevicePage}; + cudaStream_t stream = nullptr; + bool registered = false; + int device = 0; + ThreadStaging() { + check(cudaGetDevice(&device)); + int low, high; + check(cudaDeviceGetStreamPriorityRange(&low, &high)); + check( + cudaStreamCreateWithPriority(&stream, cudaStreamNonBlocking, high)); + } + ~ThreadStaging() { + cudaSetDevice(device); + if (stream) { + cudaStreamSynchronize(stream); + } + if (registered) { + cuFileBufDeregister(raw.p); + } + if (stream) { + cudaStreamDestroy(stream); + } + } + void reserve(std::size_t size) { + if (registered && size <= raw.capacity) { + return; + } + if (registered) { + auto status = cuFileBufDeregister(raw.p); + if (status.err != CU_FILE_SUCCESS) { + throw RadioError("cuFile staging deregistration failed"); + } + registered = false; + } + raw.reserve(size); + auto status = cuFileBufRegister(raw.p, raw.capacity, 0); + if (status.err != CU_FILE_SUCCESS) { + throw RadioError( + "cuFile staging registration failed: " + + std::to_string(status.err)); + } + registered = true; + } +}; +ThreadStaging& worker_staging(std::size_t bytes) { + thread_local std::unique_ptr staging; + int device; + check(cudaGetDevice(&device)); + if (staging && staging->device != device) { + staging.reset(); + check(cudaSetDevice(device)); + } + if (!staging) { + staging = std::make_unique(); + } + staging->reserve(bytes); + return *staging; +} +#endif +} // namespace +void ensure_cufile_driver() { +#ifdef XBEAM_HAVE_CUFILE + static Driver driver; +#else + throw RadioError("native CUDA build does not include cuFile"); +#endif +} +struct CudaRecordingInput::Impl { + int fd = -1; + const ReadMode mode; + const bool shared_staging; + bool gds() const { + return mode == ReadMode::GpuDirect; + } + // Direct and cuFile reads bypass the page cache and need aligned extents. + bool aligned() const { + return mode != ReadMode::Buffered; + } + std::size_t chunk_bytes; + std::int64_t framed, bytes, file_size; + gpu::Memory raw{kDevicePage}, packed{kDevicePage}, valid{kDevicePage}; + gpu::Pinned host; + cudaStream_t stream = nullptr; + std::atomic active{false}; + std::atomic failed{false}; + std::atomic remaining{0}; + std::uint64_t generation = 0; + std::int64_t read_begin = 0, read_prefix = 0, read_frames = 0; + std::vector pending_validity; + std::vector chunks; + std::unique_ptr[]> claimed; +#ifdef XBEAM_HAVE_CUFILE + CUfileHandle_t handle = nullptr; + std::vector registered_regions; +#endif + Impl( + const std::string& source, + std::int64_t frame, + std::int64_t payload, + ReadMode read_mode, + std::size_t chunk, + bool shared) + : mode(read_mode), + shared_staging(shared), + chunk_bytes(chunk), + framed(frame), + bytes(payload) { + if (frame <= 32 || payload != frame - 32) { + throw RadioError("invalid recording frame geometry"); + } + if (!chunk_bytes || chunk_bytes % 65536 || chunk_bytes > 1073741824) { + throw RadioError( + "native read chunks must be 64-KiB multiples up to 1 GiB"); + } + if (shared_staging && !gds()) { + throw RadioError("shared read staging requires cuFile"); + } +#ifndef XBEAM_HAVE_CUFILE + if (gds()) { + throw RadioError("native CUDA build does not include cuFile"); + } +#endif + fd = ::open( + source.c_str(), O_RDONLY | O_CLOEXEC | (aligned() ? O_DIRECT : 0)); + if (fd < 0) { + throw RadioError("cannot open native recording input"); + } + try { + struct stat st{}; + if (::fstat(fd, &st) || !S_ISREG(st.st_mode) || st.st_size <= 0 + || st.st_size % frame) { + throw RadioError("invalid native input file extent"); + } + file_size = st.st_size; + check(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); +#ifdef XBEAM_HAVE_CUFILE + if (gds()) { + ensure_cufile_driver(); + CUfileDescr_t descriptor{}; + descriptor.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD; + descriptor.handle.fd = fd; + auto status = cuFileHandleRegister(&handle, &descriptor); + if (status.err != CU_FILE_SUCCESS) { + throw RadioError( + "cuFile handle registration failed: " + + std::to_string(status.err)); + } + } +#endif + } catch (...) { + if (stream) { + cudaStreamDestroy(stream); + } + ::close(fd); + fd = -1; + throw; + } + } + ~Impl() { + if (stream) { + cudaStreamSynchronize(stream); + cudaStreamDestroy(stream); + } +#ifdef XBEAM_HAVE_CUFILE + for (auto region : registered_regions) { + cuFileBufDeregister(region); + } + if (handle) { + cuFileHandleDeregister(handle); + } +#endif + if (fd >= 0) { + ::close(fd); + } + } +}; +CudaRecordingInput::CudaRecordingInput( + const std::string& source, + std::int64_t frame, + std::int64_t payload, + ReadMode mode, + std::size_t chunk_bytes, + bool shared_staging) + : impl( + std::make_unique( + source, frame, payload, mode, chunk_bytes, shared_staging)) {} +CudaRecordingInput::~CudaRecordingInput() = default; +std::vector CudaRecordingInput::begin( + std::int64_t first, + std::int64_t stop, + const std::vector& validity) { + auto& state = *impl; + if (state.active.exchange(true)) { + throw RadioError("native input already has an active transfer"); + } + try { + auto& p = *impl; + if (first < 0 || stop <= first || stop > p.file_size / p.framed + || std::size_t(stop - first) != validity.size() + || __int128(stop - first) * p.framed > 1073741824) { + throw RadioError("native input read exceeds bounded extent"); + } + const auto offset = first * p.framed, + begin = p.aligned() ? offset / 65536 * 65536 : offset; + const auto end = p.aligned() + ? std::min(p.file_size, (stop * p.framed + 65535) / 65536 * 65536) + : stop * p.framed; + const auto count = std::size_t(end - begin); +#ifdef XBEAM_HAVE_CUFILE + if (p.gds() && !p.shared_staging && count > p.raw.capacity) { + while (!p.registered_regions.empty()) { + auto status = cuFileBufDeregister(p.registered_regions.back()); + if (status.err != CU_FILE_SUCCESS) { + throw RadioError("cuFile buffer deregistration failed"); + } + p.registered_regions.pop_back(); + } + } +#endif + if (!p.shared_staging) { + p.raw.reserve((count + 65535) / 65536 * 65536); + } +#ifdef XBEAM_HAVE_CUFILE + if (p.gds() && !p.shared_staging && p.registered_regions.empty()) { + p.registered_regions.reserve( + (p.raw.capacity + p.chunk_bytes - 1) / p.chunk_bytes); + try { + for (std::size_t offset = 0; offset < p.raw.capacity; + offset += p.chunk_bytes) { + auto address = + static_cast(p.raw.p) + offset; + auto size = + std::min(p.chunk_bytes, p.raw.capacity - offset); + auto status = cuFileBufRegister(address, size, 0); + if (status.err != CU_FILE_SUCCESS) { + throw RadioError( + "cuFile buffer registration failed: " + + std::to_string(status.err)); + } + p.registered_regions.push_back(address); + } + } catch (...) { + for (auto region : p.registered_regions) { + cuFileBufDeregister(region); + } + p.registered_regions.clear(); + throw; + } + } +#endif + p.packed.reserve(std::size_t(stop - first) * p.bytes); + p.valid.reserve(validity.size()); + + if (!p.gds()) { + p.host.reserve( + p.mode == ReadMode::Direct ? (count + 65535) / 65536 * 65536 + : count); + } + if (p.generation == UINT64_MAX) { + throw RadioError("native input generation overflow"); + } + ++p.generation; + p.read_begin = begin; + p.read_prefix = offset - begin; + p.read_frames = stop - first; + p.pending_validity = validity; + p.chunks.clear(); + for (std::size_t position = 0; position < count; + position += p.chunk_bytes) { + p.chunks.push_back( + {p.generation, + p.chunks.size(), + position, + std::min(p.chunk_bytes, count - position)}); + } + p.claimed = + std::make_unique[]>(p.chunks.size()); + for (std::size_t k = 0; k < p.chunks.size(); ++k) { + p.claimed[k].store(0); + } + p.remaining.store(p.chunks.size()); + p.failed.store(false); + return p.chunks; + } catch (...) { + state.active.store(false); + throw; + } +} +void CudaRecordingInput::read_chunk(const NativeReadChunk& chunk) { + auto& p = *impl; + if (!p.active.load() || chunk.generation != p.generation + || chunk.index >= p.chunks.size()) { + throw RadioError("stale native input chunk"); + } + const auto& expected = p.chunks[chunk.index]; + if (chunk.offset != expected.offset || chunk.size != expected.size + || p.claimed[chunk.index].exchange(1)) { + throw RadioError("invalid or duplicate native input chunk"); + } + struct Completion { + std::atomic& remaining; + std::atomic& failed; + bool success = false; + ~Completion() { + if (!success) { + failed.store(true); + } + remaining.fetch_sub(1); + } + } complete{p.remaining, p.failed}; + if (p.gds()) { +#ifdef XBEAM_HAVE_CUFILE + auto* staging = + p.shared_staging ? &worker_staging(p.chunk_bytes) : nullptr; + auto pointer = staging + ? staging->raw.p + : static_cast(p.raw.p) + chunk.offset; + auto n = cuFileRead( + p.handle, pointer, chunk.size, p.read_begin + chunk.offset, 0); + if (n < 0 || std::size_t(n) != chunk.size) { + throw RadioError( + "cuFile native chunk read was short or failed: " + + std::to_string(n)); + } + if (staging) { + auto first = (p.read_begin + p.read_prefix) / p.framed; + unpack_frame_chunk( + pointer, + p.packed.p, + first, + p.read_frames, + p.framed, + p.bytes, + p.read_begin + chunk.offset, + chunk.size, + reinterpret_cast(staging->stream)); + // Future completion includes extraction. High priority keeps a + // short staging operation from waiting behind a long compute grid. + check(cudaStreamSynchronize(staging->stream)); + } +#endif + } else { + std::size_t done = 0; + // O_DIRECT needs block-multiple lengths; only the last chunk can be ragged + // and the kernel returns the bytes up to EOF for it. + const std::size_t want = p.mode == ReadMode::Direct + ? (chunk.size + 4095) / 4096 * 4096 + : chunk.size; + while (done < chunk.size) { + auto n = ::pread( + p.fd, + static_cast(p.host.p) + chunk.offset + done, + want - done, + p.read_begin + chunk.offset + done); + if (n < 0 && errno == EINTR) { + continue; + } + if (n <= 0) { + throw RadioError("short native POSIX chunk read"); + } + done += n; + } + check(cudaMemcpyAsync( + static_cast(p.raw.p) + chunk.offset, + static_cast(p.host.p) + chunk.offset, + chunk.size, + cudaMemcpyHostToDevice, + p.stream)); + } + complete.success = true; +} +void CudaRecordingInput::finish() { + auto& p = *impl; + if (!p.active.load() || p.remaining.load() != 0) { + throw RadioError("unfinished native input chunks"); + } + if (p.failed.load()) { + throw RadioError("failed native input transfer cannot be published"); + } + check(cudaMemcpyAsync( + p.valid.p, + p.pending_validity.data(), + p.pending_validity.size(), + cudaMemcpyHostToDevice, + p.stream)); + if (!p.shared_staging) { + unpack_frames( + p.raw.p, + p.packed.p, + p.read_frames, + p.framed, + p.bytes, + p.read_prefix, + reinterpret_cast(p.stream)); + } + check(cudaStreamSynchronize(p.stream)); + p.active.store(false); +} +void CudaRecordingInput::read( + std::int64_t first, + std::int64_t stop, + const std::vector& validity) { + for (const auto& chunk : begin(first, stop, validity)) { + read_chunk(chunk); + } + finish(); +} +std::uintptr_t CudaRecordingInput::payload() const { + return reinterpret_cast(impl->packed.p); +} +std::uintptr_t CudaRecordingInput::validity() const { + return reinterpret_cast(impl->valid.p); +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/input.cu b/src/cuphoton/xbeam/native/cuda/input.cu new file mode 100644 index 00000000..d72033f6 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/input.cu @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/cuda.hpp" +#include "xbeam/radio.hpp" +#include +#include +namespace xbeam { +__global__ void unpack_frame_payload( + const unsigned char* raw, + unsigned char* payload, + long long total, + long long framed, + long long bytes, + long long prefix) { + for (long long index = (long long)blockIdx.x * blockDim.x + threadIdx.x; + index < total; + index += (long long)gridDim.x * blockDim.x) { + payload[index] = + raw[prefix + (index / bytes) * framed + 32 + index % bytes]; + } +} +template +__global__ void unpack_frame_payload_chunk( + const unsigned char* raw, + unsigned char* payload, + long long first, + long long destination_first, + long long total, + long long framed, + long long bytes, + long long file_offset, + long long read_bytes) { + const long long width = PayloadBytes ? PayloadBytes : bytes; + for (long long index = (long long)blockIdx.x * blockDim.x + threadIdx.x; + index < total; + index += (long long)gridDim.x * blockDim.x) { + auto source = (first + index / width) * framed + 32 + index % width; + if (source >= file_offset && source - file_offset < read_bytes) { + payload[(first - destination_first) * width + index] = + raw[source - file_offset]; + } + } +} + +void unpack_frame_chunk( + const void* raw, + void* payload, + std::int64_t destination_first, + std::int64_t frames, + std::int64_t framed, + std::int64_t bytes, + std::int64_t offset, + std::int64_t read_bytes, + std::uintptr_t stream) { + if (!raw || !payload || destination_first < 0 || frames <= 0 || framed <= 32 + || bytes != framed - 32 || offset < 0 || read_bytes <= 0 + || offset > INT64_MAX - read_bytes + || destination_first > INT64_MAX - frames + || destination_first + frames > INT64_MAX / framed) { + throw RadioError("invalid native chunk extraction scope"); + } + auto first = std::max(destination_first, offset / framed); + auto stop = std::min( + destination_first + frames, 1 + (offset + read_bytes - 1) / framed); + if (stop <= first) { + return; + } + if (stop - first > INT64_MAX / bytes) { + throw RadioError("native chunk extraction exceeds int64"); + } + auto total = (stop - first) * bytes; + cudaLaunchConfig_t config{}; + config.gridDim = dim3(std::min((total - 1) / 256 + 1, 65535)); + config.blockDim = dim3(256); + config.stream = reinterpret_cast(stream); + auto launch = [&](auto kernel) { + return cudaLaunchKernelEx( + &config, + kernel, + static_cast(raw), + static_cast(payload), + first, + destination_first, + total, + framed, + bytes, + offset, + read_bytes); + }; + auto status = bytes == 8192 ? launch(unpack_frame_payload_chunk<8192>) + : bytes == 8000 ? launch(unpack_frame_payload_chunk<8000>) + : launch(unpack_frame_payload_chunk<0>); + if (status != cudaSuccess) { + throw RadioError(cudaGetErrorString(status)); + } +} +void unpack_frames( + const void* raw, + void* payload, + std::int64_t frames, + std::int64_t framed, + std::int64_t bytes, + std::int64_t prefix, + std::uintptr_t stream) { + if (!raw || !payload || frames <= 0 || framed <= 32 || bytes != framed - 32 + || prefix < 0 || frames > INT64_MAX / bytes) { + throw RadioError("invalid native framed payload extraction"); + } + auto total = frames * bytes; + unpack_frame_payload<<< + static_cast( + std::min((total + 255) / 256, 65535)), + 256, + 0, + reinterpret_cast(stream)>>>( + static_cast(raw), + static_cast(payload), + total, + framed, + bytes, + prefix); + auto status = cudaGetLastError(); + if (status != cudaSuccess) { + throw RadioError(cudaGetErrorString(status)); + } +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/input_test.cpp b/src/cuphoton/xbeam/native/cuda/input_test.cpp new file mode 100644 index 00000000..240d414e --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/input_test.cpp @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/cuda.hpp" +#include "xbeam/work_queue.hpp" +#include +#include +#include +#include +#include +#include +#include +namespace { +void check(cudaError_t status) { + if (status != cudaSuccess) { + throw std::runtime_error(cudaGetErrorString(status)); + } +} +struct Temporary { + std::string path; + explicit Temporary(const std::string& directory) { + auto name = + (std::filesystem::path(directory) / "xbeam-native-input-XXXXXX") + .string(); + std::vector buffer(name.begin(), name.end()); + buffer.push_back(0); + int fd = ::mkstemp(buffer.data()); + if (fd < 0) { + throw std::runtime_error("cannot create native input fixture"); + } + path = buffer.data(); + ::close(fd); + } + ~Temporary() { + ::unlink(path.c_str()); + } +}; +unsigned char code(int frame, int byte) { + return (frame * 17 + byte * 7) % 251; +} +} // namespace +int main(int argc, char** argv) { + try { + check(cudaSetDevice(0)); + const std::string io = argc > 2 ? argv[2] : "posix"; + const auto mode = io == "cufile" ? xbeam::ReadMode::GpuDirect + : io == "posix-direct" ? xbeam::ReadMode::Direct + : xbeam::ReadMode::Buffered; + const bool shared = argc > 4 && std::string(argv[4]) == "shared"; + const std::size_t chunk = + argc > 3 ? std::stoull(argv[3]) : 16 * 1024 * 1024; + if (!chunk || chunk % 65536 || chunk > 1073741824) { + throw std::runtime_error("invalid test chunk size"); + } + for (int payload : {8000, 8192}) { + const int framed = payload + 32, + frames = + std::max(4097, chunk / payload + 128); + Temporary temporary(argc > 1 ? argv[1] : "/tmp"); + FILE* file = ::fopen(temporary.path.c_str(), "wb"); + if (!file) { + throw std::runtime_error("cannot write native input fixture"); + } + for (int frame = 0; frame < frames; ++frame) { + std::vector data(framed, 0xAA); + for (int byte = 0; byte < payload; ++byte) { + data[32 + byte] = code(frame, byte); + } + if (::fwrite(data.data(), 1, data.size(), file) + != data.size()) { + ::fclose(file); + throw std::runtime_error( + "short native input fixture write"); + } + } + if (::fclose(file)) { + throw std::runtime_error("native input fixture close failed"); + } + xbeam::CudaRecordingInput a( + temporary.path, framed, payload, mode, chunk, shared), + b(temporary.path, framed, payload, mode, chunk, shared); + xbeam::WorkQueue queue(2); + for (auto range : std::vector>{{1, 4}, + {0, 32}, + {31, frames}, + {frames - 17, frames}, + {20, 25}}) { + std::vector validity( + range.second - range.first, 1); + validity[0] = 0; + std::vector> pending; + for (auto* input : {&a, &b}) { + auto chunks = + input->begin(range.first, range.second, validity); + bool refused = false; + try { + input->finish(); + } catch (const std::exception&) { + refused = true; + } + if (!refused) { + throw std::runtime_error( + "input finished before its chunks"); + } + if (range.second == frames && range.first == 31 + && chunks.size() < 2) { + throw std::runtime_error( + "fixture did not cross read chunks"); + } + for (const auto& chunk : chunks) { + pending.push_back(queue.submit([input, chunk] { + check(cudaSetDevice(0)); + input->read_chunk(chunk); + })); + } + } + for (auto& future : pending) { + future.wait(); + } + for (auto& future : pending) { + future.get(); + } + a.finish(); + b.finish(); + for (auto* input : {&a, &b}) { + std::vector actual( + validity.size() * payload), + mask(validity.size()); + check(cudaMemcpy( + actual.data(), + reinterpret_cast(input->payload()), + actual.size(), + cudaMemcpyDeviceToHost)); + check(cudaMemcpy( + mask.data(), + reinterpret_cast(input->validity()), + mask.size(), + cudaMemcpyDeviceToHost)); + if (mask != validity) { + throw std::runtime_error( + "native validity upload mismatch"); + } + for (int frame = range.first; frame < range.second; + ++frame) { + for (int byte = 0; byte < payload; ++byte) { + if (actual[(frame - range.first) * payload + byte] + != code(frame, byte)) { + throw std::runtime_error( + "native framed payload mismatch"); + } + } + } + } + } + // A failed read must never publish the partially populated buffer, + // even after all chunk tasks have returned. + xbeam::CudaRecordingInput broken( + temporary.path, framed, payload, mode, chunk, shared); + auto chunks = + broken.begin(0, 32, std::vector(32, 1)); + if (::truncate(temporary.path.c_str(), 0)) { + throw std::runtime_error("cannot truncate failed-read fixture"); + } + bool read_failed = false, publication_failed = false; + try { + broken.read_chunk(chunks.at(0)); + } catch (const std::exception&) { + read_failed = true; + } + try { + broken.finish(); + } catch (const std::exception&) { + publication_failed = true; + } + if (!read_failed || !publication_failed) { + throw std::runtime_error("failed native read was published"); + } + std::cout << "payload=" << payload << " mode=" << io + << " chunk=" << chunk << " shared=" << shared + << " concurrent resize/unaligned/EOF delivery PASS\n"; + } + return 0; + } catch (const std::exception& error) { + std::cerr << "ERROR: " << error.what() << '\n'; + return 1; + } +} diff --git a/src/cuphoton/xbeam/native/cuda/main.cpp b/src/cuphoton/xbeam/native/cuda/main.cpp new file mode 100644 index 00000000..4a88f0c0 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/main.cpp @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/execution.hpp" +#include +int main(int argc, char** argv) { + try { + std::vector arguments; + for (int i = 0; i < argc; ++i) { + arguments.emplace_back(argv[i]); + } + std::cout << xbeam::correlate_radio(arguments) << '\n'; + return 0; + } catch (const std::exception& e) { + std::cerr << "ERROR: " << e.what() << '\n'; + return 1; + } +} diff --git a/src/cuphoton/xbeam/native/cuda/multi_products.cu b/src/cuphoton/xbeam/native/cuda/multi_products.cu new file mode 100644 index 00000000..e67c70f7 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/multi_products.cu @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "multi_products.inc" +#undef BAND_LIMIT +#undef PRODUCT_LIMIT +namespace narrow_products { +#define BAND_LIMIT 4 +#define PRODUCT_LIMIT 4 +#define batched_visibility_products_float batched_visibility_products_float_4x4 +#include "multi_products.inc" +#undef batched_visibility_products_float +#undef PRODUCT_LIMIT +#undef BAND_LIMIT +} // namespace narrow_products +namespace wide_products { +#define BAND_LIMIT 8 +#define PRODUCT_LIMIT 4 +#define batched_visibility_products_float batched_visibility_products_float_8x4 +#include "multi_products.inc" +#undef batched_visibility_products_float +#undef PRODUCT_LIMIT +#undef BAND_LIMIT +} // namespace wide_products +#include "xbeam/cuda.hpp" +#include "xbeam/radio.hpp" +#include +#include +#include +#include +#include +namespace xbeam { +namespace { +void check(cudaError_t status) { + if (status != cudaSuccess) { + throw RadioError(cudaGetErrorString(status)); + } +} +struct Memory { + void* p = nullptr; + ~Memory() { + if (p) { + cudaFree(p); + } + } + template + void upload(const std::vector& values) { + check(cudaMalloc(&p, values.size() * sizeof(T))); + check(cudaMemcpy( + p, + values.data(), + values.size() * sizeof(T), + cudaMemcpyHostToDevice)); + } +}; +struct Group { + int channels, averaging, packs, band_limit = 0; + Memory spectra, strides, left, right, band_counts, product_counts, rows, wa, + wb, multiply, normalization; +}; +bool same(const VisibilityProduct& a, const VisibilityProduct& b) { + return a.spectrum_a == b.spectrum_a && a.spectrum_b == b.spectrum_b + && a.weight_a == b.weight_a && a.weight_b == b.weight_b + && a.stride_a == b.stride_a && a.stride_b == b.stride_b + && a.channels == b.channels && a.averaging == b.averaging + && a.inverse_normalization == b.inverse_normalization + && a.multiply_weights == b.multiply_weights; +} +} // namespace +struct CudaProducts::Impl { + std::vector previous; + std::vector> groups; + std::vector singles; + std::uintptr_t stream = 0; + int product_limit; + bool reuse_sources; +}; +CudaProducts::CudaProducts(int products_per_pack, bool reuse_sources) + : impl(std::make_unique()) { + if (products_per_pack != 4 && products_per_pack != 8) { + throw RadioError( + "native XMAC packs must contain four or eight products"); + } + impl->product_limit = products_per_pack; + impl->reuse_sources = reuse_sources; +} +CudaProducts::~CudaProducts() { + cudaStreamSynchronize(reinterpret_cast(impl->stream)); +} +void CudaProducts::configure(const std::vector& products) { + auto& state = *impl; + if (products.size() == state.previous.size() + && std::equal( + products.begin(), products.end(), state.previous.begin(), same)) { + return; + } + check(cudaStreamSynchronize(reinterpret_cast(state.stream))); + auto previous = products; + std::vector> groups; + std::vector singles; + std::map, std::vector> compatible; + for (std::size_t k = 0; k < products.size(); ++k) { + const auto& p = products[k]; + if (!p.spectrum_a || !p.spectrum_b || !p.weight_a || !p.weight_b + || p.channels <= 0 || p.averaging <= 0 || p.channels % p.averaging + || p.stride_a < p.channels || p.stride_b < p.channels + || !std::isfinite(p.inverse_normalization) + || p.inverse_normalization <= 0) { + throw RadioError("invalid prepared visibility product"); + } + if (p.averaging == 1 || p.averaging == 4 || p.averaging == 32) { + compatible[{p.channels, p.averaging}].push_back(k); + } else { + singles.push_back(k); + } + } + for (const auto& entry : compatible) { + auto group = std::make_unique(); + group->channels = entry.first.first; + group->averaging = entry.first.second; + group->packs = 0; + std::vector spectra, wa, wb; + std::vector strides; + std::vector left, right, band_counts, product_counts, rows; + std::vector multiply; + std::vector normalizations; + using Band = std::pair; + std::set remaining(entry.second.begin(), entry.second.end()); + std::map> incident; + if (state.reuse_sources) { + for (int index : entry.second) { + const auto& p = products[index]; + incident[{p.spectrum_a, p.stride_a}].push_back(index); + incident[{p.spectrum_b, p.stride_b}].push_back(index); + } + } + while (!remaining.empty()) { + std::vector bands; + std::vector items, ls, rs; + std::set nearby; + auto insert_band = [&](Band band) { + auto it = std::find(bands.begin(), bands.end(), band); + if (it != bands.end()) { + return int(it - bands.begin()); + } + bands.push_back(band); + if (state.reuse_sources) { + for (int index : incident.at(band)) { + if (remaining.count(index)) { + nearby.insert(index); + } + } + } + return int(bands.size() - 1); + }; + auto added_bands = [&](int index) { + const auto& p = products[index]; + Band a{p.spectrum_a, p.stride_a}, b{p.spectrum_b, p.stride_b}; + return int(std::find(bands.begin(), bands.end(), a) + == bands.end()) + + int( + b != a + && std::find(bands.begin(), bands.end(), b) + == bands.end()); + }; + while (!remaining.empty() + && items.size() < std::size_t(state.product_limit)) { + int id = *remaining.begin(), extra = added_bands(id); + if (state.reuse_sources) { + // Search only the local source graph, independent of the + // number of unrelated frequencies or baseline products. + int best = 3; + for (int candidate : nearby) { + int added = added_bands(candidate); + if (bands.size() + added <= 8 && added < best) { + id = candidate; + extra = best = added; + if (!added) { + break; + } + } + } + } + const auto& p = products[id]; + Band a{p.spectrum_a, p.stride_a}, b{p.spectrum_b, p.stride_b}; + if (bands.size() + extra > 8) { + break; + } + ls.push_back(insert_band(a)); + rs.push_back(insert_band(b)); + items.push_back(id); + remaining.erase(id); + nearby.erase(id); + } + if (items.empty()) { + throw RadioError("visibility pack cannot fit its source bands"); + } + ++group->packs; + group->band_limit = std::max(group->band_limit, int(bands.size())); + band_counts.push_back(bands.size()); + product_counts.push_back(items.size()); + for (std::size_t k = 0; k < 8; ++k) { + spectra.push_back(k < bands.size() ? bands[k].first : 0); + strides.push_back(k < bands.size() ? bands[k].second : 0); + if (k < items.size()) { + const auto& p = products[items[k]]; + left.push_back(ls[k]); + right.push_back(rs[k]); + rows.push_back(items[k]); + wa.push_back(p.weight_a); + wb.push_back(p.weight_b); + multiply.push_back(p.multiply_weights); + normalizations.push_back(p.inverse_normalization); + } else { + left.push_back(0); + right.push_back(0); + rows.push_back(0); + wa.push_back(0); + wb.push_back(0); + multiply.push_back(0); + normalizations.push_back(1); + } + } + } + group->spectra.upload(spectra); + group->strides.upload(strides); + group->left.upload(left); + group->right.upload(right); + group->band_counts.upload(band_counts); + group->product_counts.upload(product_counts); + group->rows.upload(rows); + group->wa.upload(wa); + group->wb.upload(wb); + group->multiply.upload(multiply); + group->normalization.upload(normalizations); + groups.push_back(std::move(group)); + } + state.groups.swap(groups); + state.singles.swap(singles); + state.previous.swap(previous); +} +void CudaProducts::submit( + int groups, + std::uintptr_t starts, + std::uintptr_t lengths, + std::uintptr_t offsets, + std::uintptr_t weight_offsets, + std::uintptr_t output, + std::uintptr_t weights, + std::uintptr_t stream) { + if (groups <= 0 || groups > 65535 || !starts || !lengths || !offsets + || !weight_offsets || !output || !weights) { + throw RadioError("invalid prepared XMAC group submission"); + } + impl->stream = stream; + auto work = reinterpret_cast(stream); + for (const auto& pointer : impl->groups) { + const auto& g = *pointer; + for (int begin = 0; begin < g.packs; begin += 65535) { + int packs = std::min(g.packs - begin, 65535); + cudaLaunchConfig_t config{}; + config.gridDim = dim3((g.channels + 31) / 32, groups, packs); + config.blockDim = dim3(256); + config.stream = work; + auto launch = [&](auto kernel) { + return cudaLaunchKernelEx( + &config, + kernel, + static_cast(g.spectra.p), + static_cast(g.strides.p), + static_cast(g.left.p), + static_cast(g.right.p), + static_cast(g.band_counts.p), + static_cast(g.product_counts.p), + static_cast(g.rows.p), + static_cast(g.wa.p), + static_cast(g.wb.p), + static_cast(g.multiply.p), + reinterpret_cast(starts), + reinterpret_cast(lengths), + reinterpret_cast(offsets), + reinterpret_cast(weight_offsets), + static_cast(g.normalization.p), + groups, + g.channels, + g.averaging, + begin, + reinterpret_cast(output), + reinterpret_cast(weights)); + }; + if (impl->product_limit == 8) { + check(launch(batched_visibility_products_float)); + } else if (g.band_limit <= 4) { + check(launch( + narrow_products::batched_visibility_products_float_4x4)); + } else { + check(launch( + wide_products::batched_visibility_products_float_8x4)); + } + } + } + for (int index : impl->singles) { + const auto& p = impl->previous[index]; + ProductLayout layout{p.channels, + p.averaging, + groups, + p.stride_a, + p.stride_b, + p.inverse_normalization, + p.multiply_weights}; + submit_product( + layout, + p.spectrum_a, + p.spectrum_b, + p.weight_a, + p.weight_b, + starts, + lengths, + offsets + std::size_t(index) * groups * 8, + weight_offsets + std::size_t(index) * groups * 8, + output, + weights, + stream); + } +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/products.cu b/src/cuphoton/xbeam/native/cuda/products.cu new file mode 100644 index 00000000..7c32b96d --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/products.cu @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "products.inc" +#include "xbeam/cuda.hpp" +#include "xbeam/radio.hpp" +#include +namespace xbeam { +void submit_product( + const ProductLayout& g, + std::uintptr_t aa, + std::uintptr_t bb, + std::uintptr_t wa, + std::uintptr_t wb, + std::uintptr_t starts, + std::uintptr_t lengths, + std::uintptr_t offsets, + std::uintptr_t weight_offsets, + std::uintptr_t output, + std::uintptr_t weights, + std::uintptr_t stream) { + if (g.channels <= 0 || g.averaging <= 0 || g.channels % g.averaging + || g.groups <= 0 || g.groups > 65535 || g.stride_a < g.channels + || g.stride_b < g.channels || !std::isfinite(g.inverse_normalization) + || g.inverse_normalization <= 0 || !aa || !bb || !wa || !wb || !starts + || !lengths || !offsets || !weight_offsets || !output || !weights) { + throw RadioError("invalid native XMAC submission"); + } + const auto* a = reinterpret_cast(aa); + const auto* b = reinterpret_cast(bb); + const auto* weight_a = reinterpret_cast(wa); + const auto* weight_b = reinterpret_cast(wb); + const auto* first = reinterpret_cast(starts); + const auto* count = reinterpret_cast(lengths); + const auto* position = reinterpret_cast(offsets); + const auto* wp = reinterpret_cast(weight_offsets); + auto* result = reinterpret_cast(output); + auto* exposure = reinterpret_cast(weights); + const auto work = reinterpret_cast(stream); + const dim3 grid((g.channels + 31) / 32, g.groups); + switch (g.averaging) { + case 1: + batched_visibility_product_float_avg1<<>>( + a, + b, + g.stride_a, + g.stride_b, + weight_a, + weight_b, + g.multiply_weights, + first, + count, + position, + wp, + g.channels, + g.inverse_normalization, + result, + exposure); + break; + case 4: + batched_visibility_product_float_avg4<<>>( + a, + b, + g.stride_a, + g.stride_b, + weight_a, + weight_b, + g.multiply_weights, + first, + count, + position, + wp, + g.channels, + g.inverse_normalization, + result, + exposure); + break; + case 32: + batched_visibility_product_float_avg32<<>>( + a, + b, + g.stride_a, + g.stride_b, + weight_a, + weight_b, + g.multiply_weights, + first, + count, + position, + wp, + g.channels, + g.inverse_normalization, + result, + exposure); + break; + default: + batched_visibility_product_float_generic<<< + dim3((g.channels / g.averaging + 255) / 256, g.groups), + 256, + 0, + work>>>( + a, + b, + g.stride_a, + g.stride_b, + weight_a, + weight_b, + g.multiply_weights, + first, + count, + position, + wp, + g.channels, + g.averaging, + g.inverse_normalization, + result, + exposure); + } + auto status = cudaGetLastError(); + if (status != cudaSuccess) { + throw RadioError(cudaGetErrorString(status)); + } +} +} // namespace xbeam + +namespace xbeam { +void sum_series_spectra( + bool wide, + int rows, + int channels, + const void* a, + const void* b, + const double* weights, + double scale, + void* output, + double* exposure, + std::uintptr_t stream) { + const auto work = reinterpret_cast(stream); + const dim3 grid((channels + 31) / 32); + if (wide) { + visibility_product_double_avg1<<>>( + static_cast(a), + static_cast(b), + weights, + weights, + false, + rows, + channels, + scale, + static_cast(output), + exposure); + } else { + visibility_product_float_avg1<<>>( + static_cast(a), + static_cast(b), + weights, + weights, + false, + rows, + channels, + scale, + static_cast(output), + exposure); + } + auto status = cudaGetLastError(); + if (status != cudaSuccess) { + throw RadioError(cudaGetErrorString(status)); + } +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/products_test.cpp b/src/cuphoton/xbeam/native/cuda/products_test.cpp new file mode 100644 index 00000000..3bb54092 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/products_test.cpp @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/cuda.hpp" +#include +#include +#include +#include +#include +namespace { +void check(cudaError_t c) { + if (c != cudaSuccess) { + throw std::runtime_error(cudaGetErrorString(c)); + } +} +struct Buffer { + void* p = nullptr; + explicit Buffer(std::size_t n) { + check(cudaMalloc(&p, n)); + } + ~Buffer() { + if (p) { + cudaFree(p); + } + } + std::uintptr_t address() { + return reinterpret_cast(p); + } +}; +} // namespace +int main() { + try { + for (bool reuse : {false, true}) { + for (bool narrow : {false, true}) { + for (int pack_products : {8, 4}) { + for (int avg : {1, 4, 32, 7}) { + const int rows = 8, channels = 224, sa = 229, sb = 231, + out = channels / avg; + std::vector a(rows * sa * 2, 0), + b(rows * sb * 2, 0); + for (int row = 0; row < rows; ++row) { + for (int c = 0; c < sa; ++c) { + a[(row * sa + c) * 2] = 2; + a[(row * sa + c) * 2 + 1] = 3; + } + } + for (int row = 0; row < rows; ++row) { + for (int c = 0; c < sb; ++c) { + b[(row * sb + c) * 2] = 4; + b[(row * sb + c) * 2 + 1] = -1; + } + } + std::vector wa(rows, .5), wb(rows, .25); + int starts[] = {0, 4}, lengths[] = {4, 4}; + long long offsets[] = {7, 7 + out}, + weight_offsets[] = {2, 3}; + Buffer da(a.size() * 4), db(b.size() * 4), + dwa(rows * 8), dwb(rows * 8), ds(sizeof(starts)), + dl(sizeof(lengths)), doo(sizeof(offsets)), + dwo(sizeof(weight_offsets)), + result((7 + out * 2) * 8), exposure(4 * 8); + check(cudaMemcpy( + da.p, + a.data(), + a.size() * 4, + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + db.p, + b.data(), + b.size() * 4, + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + dwa.p, + wa.data(), + rows * 8, + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + dwb.p, + wb.data(), + rows * 8, + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + ds.p, + starts, + sizeof(starts), + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + dl.p, + lengths, + sizeof(lengths), + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + doo.p, + offsets, + sizeof(offsets), + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + dwo.p, + weight_offsets, + sizeof(weight_offsets), + cudaMemcpyHostToDevice)); + xbeam::ProductLayout g{ + channels, avg, 2, sa, sb, .5, true}; + xbeam::submit_product( + g, + da.address(), + db.address(), + dwa.address(), + dwb.address(), + ds.address(), + dl.address(), + doo.address(), + dwo.address(), + result.address(), + exposure.address(), + 0); + std::vector host((7 + out * 2) * 2); + double weights[4]; + check(cudaMemcpy( + host.data(), + result.p, + host.size() * 4, + cudaMemcpyDeviceToHost)); + check(cudaMemcpy( + weights, + exposure.p, + sizeof(weights), + cudaMemcpyDeviceToHost)); + for (int group = 0; group < 2; ++group) { + if (weights[2 + group] != .5) { + throw std::runtime_error( + "XMAC weight mismatch"); + } + for (int c = 0; c < out; ++c) { + auto index = (7 + group * out + c) * 2; + if (host[index] != 10 + || host[index + 1] != 28) { + throw std::runtime_error( + "XMAC spectrum mismatch"); + } + } + } + // Distinct rows/channels make incorrect source reuse and + // output permutations visible to the independent CPU sum. + for (int row = 0; row < rows; ++row) { + for (int c = 0; c < sa; ++c) { + a[(row * sa + c) * 2] = + 2 + row / 32.f + c / 256.f; + a[(row * sa + c) * 2 + 1] = + 3 + c / 128.f - row / 64.f; + } + for (int c = 0; c < sb; ++c) { + b[(row * sb + c) * 2] = + 4 - row / 64.f + c / 512.f; + b[(row * sb + c) * 2 + 1] = + -1 + row / 128.f - c / 1024.f; + } + } + check(cudaMemcpy( + da.p, + a.data(), + a.size() * 4, + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + db.p, + b.data(), + b.size() * 4, + cudaMemcpyHostToDevice)); + const int product_count = 17; + std::vector products; + std::vector packed_offsets, + packed_weight_offsets; + for (int product = 0; product < product_count; + ++product) { + const auto aa = da.address() + + (narrow ? 0 : product % 5) * 8, + bb = db.address() + + (narrow ? 0 : product % 7) * 8; + bool swap = product % 2; + products.push_back( + {swap ? bb : aa, + swap ? aa : bb, + swap ? dwb.address() : dwa.address(), + swap ? dwa.address() : dwb.address(), + swap ? sb : sa, + swap ? sa : sb, + channels, + avg, + product % 3 ? .5 : .25, + !swap}); + for (int group = 0; group < 2; ++group) { + packed_offsets.push_back( + 7 + (product * 2 + group) * out); + packed_weight_offsets.push_back( + 2 + product * 2 + group); + } + } + Buffer po(packed_offsets.size() * 8), + pw(packed_weight_offsets.size() * 8), + packed_result((7 + product_count * 2 * out) * 8), + packed_weights((2 + product_count * 2) * 8); + check(cudaMemcpy( + po.p, + packed_offsets.data(), + packed_offsets.size() * 8, + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + pw.p, + packed_weight_offsets.data(), + packed_weight_offsets.size() * 8, + cudaMemcpyHostToDevice)); + auto expected = [&](int product, + int group, + int channel) { + double real = 0, imag = 0; + for (int row = starts[group]; + row < starts[group] + lengths[group]; + ++row) { + for (int c = channel * avg; + c < (channel + 1) * avg; + ++c) { + auto ai = + (row * sa + (narrow ? 0 : product % 5) + + c) + * 2; + auto bi = + (row * sb + (narrow ? 0 : product % 7) + + c) + * 2; + double ar = a[ai], av = a[ai + 1], + br = b[bi], bv = b[bi + 1]; + real += ar * br + av * bv; + imag += av * br - ar * bv; + } + } + double scale = + products[product].inverse_normalization / avg; + return std::make_pair( + real * scale, + imag * scale * (product % 2 ? -1 : 1)); + }; + xbeam::CudaProducts prepared(pack_products, reuse); + prepared.configure(products); + prepared.configure(products); + auto invalid_products = products; + invalid_products.back().channels = 0; + for (int attempt = 0; attempt < 2; ++attempt) { + bool rejected = false; + try { + prepared.configure(invalid_products); + } catch (const std::exception&) { + rejected = true; + } + if (!rejected) { + throw std::runtime_error( + "invalid prepared configuration accepted"); + } + } + prepared.submit( + 2, + ds.address(), + dl.address(), + po.address(), + pw.address(), + packed_result.address(), + packed_weights.address(), + 0); + std::vector packed_host( + (7 + product_count * 2 * out) * 2); + std::vector packed_exposure( + 2 + product_count * 2); + check(cudaMemcpy( + packed_host.data(), + packed_result.p, + packed_host.size() * 4, + cudaMemcpyDeviceToHost)); + check(cudaMemcpy( + packed_exposure.data(), + packed_weights.p, + packed_exposure.size() * 8, + cudaMemcpyDeviceToHost)); + for (int product = 0; product < product_count; + ++product) { + for (int group = 0; group < 2; ++group) { + double expected_weight = product % 2 ? 1. : .5; + if (packed_exposure[2 + product * 2 + group] + != expected_weight) { + throw std::runtime_error( + "prepared XMAC weight mismatch"); + } + for (int c = 0; c < out; ++c) { + auto index = + (7 + (product * 2 + group) * out + c) + * 2; + auto truth = expected(product, group, c); + if (!std::isfinite(packed_host[index]) + || !std::isfinite( + packed_host[index + 1]) + || std::abs( + packed_host[index] - truth.first) + > 5e-4 + || std::abs( + packed_host[index + 1] + - truth.second) + > 5e-4) { + throw std::runtime_error( + "prepared XMAC spectrum mismatch"); + } + } + } + } + products[0].inverse_normalization = .5; + prepared.configure(products); + prepared.submit( + 2, + ds.address(), + dl.address(), + po.address(), + pw.address(), + packed_result.address(), + packed_weights.address(), + 0); + check(cudaMemcpy( + packed_host.data(), + packed_result.p, + packed_host.size() * 4, + cudaMemcpyDeviceToHost)); + auto refreshed = expected(0, 0, 0); + if (!std::isfinite(packed_host[14]) + || !std::isfinite(packed_host[15]) + || std::abs(packed_host[14] - refreshed.first) + > 5e-4 + || std::abs(packed_host[15] - refreshed.second) + > 5e-4) { + throw std::runtime_error( + "prepared XMAC descriptor refresh mismatch"); + } + std::cout + << "averaging=" << avg + << " products/pack=" << pack_products + << " narrow=" << narrow << " reuse=" << reuse + << " strided grouped and prepared XMAC PASS\n"; + } + } + } + } + return 0; + } catch (const std::exception& e) { + std::cerr << "ERROR: " << e.what() << '\n'; + return 1; + } +} diff --git a/src/cuphoton/xbeam/native/cuda/radio_model.cpp b/src/cuphoton/xbeam/native/cuda/radio_model.cpp new file mode 100644 index 00000000..0a6d3160 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/radio_model.cpp @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "radio_model.hpp" +#include +#include +#include +namespace xbeam { + +const Json& by_id(const Json& rows, int identifier) { + for (const auto& row : rows) { + if (row.at("identifier") == identifier) { + return row; + } + } + throw RadioError("unknown execution identifier"); +} + +std::unique_ptr open_index( + const std::string& source, const std::filesystem::path& directory) { + std::vector candidates; + const auto base = std::filesystem::path(source).filename().string(); + for (const auto& entry : std::filesystem::directory_iterator(directory)) { + const auto name = entry.path().filename().string(); + if (name.rfind(base + ".", 0) == 0 + && entry.path().extension() == ".vdix") { + candidates.push_back(entry.path()); + } + } + std::sort(candidates.begin(), candidates.end()); + std::string error; + for (const auto& path : candidates) { + try { + auto index = std::make_unique(source, path.string()); + if (Json::parse(index->metadata_json()).at("schema_version") != 4) { + throw RadioError( + "native execution requires a schema-v4 sidecar"); + } + return index; + } catch (const std::exception& e) { + error = e.what(); + } + } + throw RadioError("no valid sidecar for " + source + ": " + error); +} + +View view(const Json& job, int stream, int band) { + const auto& stream_row = by_id(job.at("datastreams"), stream); + const int station = stream_row.at("station_id"); + for (const auto& recorded : stream_row.at("recorded_bands")) { + if (recorded.at("identifier") == band) { + return {{stream, band}, + 0, + by_id(job.at("frequencies"), recorded.at("frequency_id")) + .at("channel_count"), + station, + recorded.at("polarization").get().at(0)}; + } + } + for (const auto& zoom : stream_row.at("zoom_bands")) { + if (zoom.at("identifier") == band) { + return {{stream, zoom.at("parent_band_id")}, + zoom.at("start_channel"), + zoom.at("channel_count"), + station, + zoom.at("polarization").get().at(0)}; + } + } + throw RadioError("unknown execution band"); +} + +std::vector products(const Json& job) { + std::vector result; + std::map> autos; + auto append = [&](int stream_a, + int band_a, + int stream_b, + int band_b, + int frequency, + bool automatic) { + const auto a = view(job, stream_a, band_a), + b = view(job, stream_b, band_b); + const auto& row = by_id(job.at("frequencies"), frequency); + const int channels = row.at("channel_count"), + average = row.at("channels_to_average"); + if (a.channels != channels || b.channels != channels || channels <= 0 + || average <= 0 || channels % average) { + throw RadioError("visibility channel geometries differ"); + } + result.push_back({a, b, frequency, channels, average, automatic, 0}); + }; + for (const auto& baseline : job.at("baselines")) { + for (const auto& p : baseline.at("products")) { + const int a = baseline.at("datastream_a_id"), + b = baseline.at("datastream_b_id"), + frequency = p.at("output_frequency_id"), + band_a = p.at("band_a_id"), band_b = p.at("band_b_id"); + append(a, band_a, b, band_b, frequency, false); + autos[{a, frequency}].insert(band_a); + autos[{b, frequency}].insert(band_b); + } + } + if (job.at("write_autocorrelations").get()) { + for (const auto& [owner, bands] : autos) { + for (int a : bands) { + for (int b : bands) { + append(owner.first, a, owner.first, b, owner.second, true); + } + } + } + } + std::size_t offset = 0; + for (auto& p : result) { + p.offset = offset; + offset += p.channels / p.average; + } + return result; +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/radio_model.hpp b/src/cuphoton/xbeam/native/cuda/radio_model.hpp new file mode 100644 index 00000000..e1e20b56 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/radio_model.hpp @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 +// Views of the normalized job: bands, visibility products, and sidecar lookup. +#pragma once +#include "xbeam/radio.hpp" +#include +#include +#include +#include +#include +#include +namespace xbeam { + +using Json = nlohmann::json; +// (datastream id, recorded band id): one station FFT pipeline. +using Key = std::pair; + +// Row of `rows` whose "identifier" equals `identifier`. +const Json& by_id(const Json& rows, int identifier); + +// A recorded band or a zoom window into one. +struct View { + Key parent; + int start, channels, station; + char polarization; +}; +View view(const Json& job, int stream, int band); + +struct Product { + View a, b; + int frequency, channels, average; + bool automatic; + std::size_t offset; // first output channel of this product +}; +// Cross products of every baseline plus, when requested, the autocorrelations. +std::vector products(const Json& job); + +// First valid schema-v4 sidecar `.*.vdix` in `directory`. +std::unique_ptr open_index( + const std::string& source, const std::filesystem::path& directory); + +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/radio_options.cpp b/src/cuphoton/xbeam/native/cuda/radio_options.cpp new file mode 100644 index 00000000..33ae64a3 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/radio_options.cpp @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "radio_options.hpp" +#include "xbeam/radio.hpp" +#include +#include +#include +namespace xbeam { +namespace { +using Raw = std::map; + +const std::string& required(const Raw& raw, const char* key) { + auto found = raw.find(key); + if (found == raw.end()) { + throw RadioError(std::string("missing execution option ") + key); + } + return found->second; +} +std::string text(const Raw& raw, const char* key, const char* fallback) { + auto found = raw.find(key); + return found == raw.end() ? fallback : found->second; +} +std::int64_t parse_integer(const std::string& value, const char* key) { + const auto error = std::string("invalid native integer option ") + key; + std::size_t consumed = 0; + std::int64_t parsed; + try { + parsed = std::stoll(value, &consumed); + } catch (const std::invalid_argument&) { + throw RadioError(error); + } catch (const std::out_of_range&) { + throw RadioError(error); + } + if (consumed != value.size()) { + throw RadioError(error); + } + return parsed; +} +std::int64_t integer(const Raw& raw, const char* key, std::int64_t fallback) { + auto found = raw.find(key); + return found == raw.end() ? fallback : parse_integer(found->second, key); +} +int narrow_integer(const Raw& raw, const char* key, int fallback) { + const auto value = integer(raw, key, fallback); + if (value < std::numeric_limits::min() + || value > std::numeric_limits::max()) { + throw RadioError( + std::string("native integer option exceeds int: ") + key); + } + return static_cast(value); +} +bool toggle(const Raw& raw, const char* key, const char* what) { + const auto value = text(raw, key, "on"); + if (value != "on" && value != "off") { + throw RadioError(std::string("native ") + what + " must be on or off"); + } + return value == "on"; +} +} // namespace + +const char* io_name(ReadMode mode) { + switch (mode) { + case ReadMode::Buffered: + return "posix"; + case ReadMode::Direct: + return "posix-direct"; + case ReadMode::GpuDirect: + return "cufile"; + } + return "?"; +} + +Options Options::parse(const std::vector& argv) { + if (argv.size() < 3) { + throw RadioError( + "usage: xbeam-native-correlate JOB.input --output DIR --index-dir DIR " + "--scan ID --first-integration N --integration-count N --fft-tile N " + "--device N --profile interop-real-usb-buffered-window-replay-v1"); + } + Raw raw; + for (std::size_t i = 2; i < argv.size(); i += 2) { + if (i + 1 >= argv.size() || !raw.emplace(argv[i], argv[i + 1]).second) { + throw RadioError("missing or repeated execution option"); + } + } + static const std::set allowed = {"--output", + "--index-dir", + "--scan", + "--first-integration", + "--integration-count", + "--fft-tile", + "--device", + "--profile", + "--vdif-io", + "--upload-workers", + "--read-chunk-bytes", + "--read-staging", + "--xmac-pack-products", + "--xmac-reuse-sources", + "--prefetch", + "--claim-dir", + "--workers-total", + "--min-chunk"}; + for (const auto& option : raw) { + if (!allowed.count(option.first)) { + throw RadioError( + "unsupported native execution option: " + option.first); + } + } + + Options o; + o.job = argv[1]; + o.profile = required(raw, "--profile"); + if (o.profile != "interop-real-usb-buffered-window-replay-v1") { + throw RadioError( + "this native runner requires the buffered-window replay contract"); + } + o.output = required(raw, "--output"); + o.index_dir = required(raw, "--index-dir"); + o.scan = required(raw, "--scan"); + o.device = narrow_integer(raw, "--device", 0); + + const auto io = text(raw, "--vdif-io", "posix"); + if (io == "posix") { + o.read_mode = ReadMode::Buffered; + } else if (io == "posix-direct") { + o.read_mode = ReadMode::Direct; + } else if (io == "cufile") { + o.read_mode = ReadMode::GpuDirect; + } else { + throw RadioError( + "native VDIF IO must be posix, posix-direct or cufile"); + } + o.prefetch = toggle(raw, "--prefetch", "prefetch"); + o.chunk_bytes = integer(raw, "--read-chunk-bytes", o.chunk_bytes); + if (o.chunk_bytes <= 0 || o.chunk_bytes % 65536 + || o.chunk_bytes > 1073741824) { + throw RadioError( + "native read chunks must be 64-KiB multiples up to 1 GiB"); + } + const auto staging = text(raw, "--read-staging", "per-input"); + o.shared_staging = staging == "shared"; + if ((staging != "per-input" && !o.shared_staging) + || (o.shared_staging && o.read_mode != ReadMode::GpuDirect)) { + throw RadioError( + "native read staging must be per-input or shared cuFile"); + } + + o.first_integration = integer(raw, "--first-integration", 0); + o.integration_count = parse_integer( + required(raw, "--integration-count"), "--integration-count"); + o.fft_tile = integer(raw, "--fft-tile", o.fft_tile); + o.upload_workers = + narrow_integer(raw, "--upload-workers", o.upload_workers); + if (o.upload_workers < 1 || o.upload_workers > 64) { + throw RadioError("native upload workers must be between 1 and 64"); + } + o.xmac_pack_products = + narrow_integer(raw, "--xmac-pack-products", o.xmac_pack_products); + o.xmac_reuse_sources = + toggle(raw, "--xmac-reuse-sources", "XMAC source reuse"); + + o.claim_dir = text(raw, "--claim-dir", ""); + if (!o.dynamic() + && (raw.count("--workers-total") || raw.count("--min-chunk"))) { + throw RadioError("chunk options require --claim-dir"); + } + o.workers_total = narrow_integer(raw, "--workers-total", 1); + o.min_chunk = integer(raw, "--min-chunk", 4); + if (o.dynamic() + && (o.workers_total < 1 || o.workers_total > 4096 || o.min_chunk < 1)) { + throw RadioError("invalid dynamic chunk scheduling options"); + } + return o; +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/radio_options.hpp b/src/cuphoton/xbeam/native/cuda/radio_options.hpp new file mode 100644 index 00000000..4fe5d5f6 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/radio_options.hpp @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 +// Command-line options of the native radio runner, parsed and validated once. +#pragma once +#include "xbeam/cuda.hpp" +#include +#include +#include +namespace xbeam { + +struct Options { + std::string job, output, index_dir, scan, profile; + std::int64_t first_integration = 0, integration_count = 0, fft_tile = 1250; + int device = 0; + ReadMode read_mode = ReadMode::Buffered; + bool shared_staging = false; + bool prefetch = true; + std::int64_t chunk_bytes = 16 * 1024 * 1024; + int upload_workers = 4; + int xmac_pack_products = 4; + bool xmac_reuse_sources = true; + // Dynamic scheduling: workers claim guided-size chunks through `claim_dir`. + std::string claim_dir; // empty = static, single-output mode + int workers_total = 1; + std::int64_t min_chunk = 4; + + bool dynamic() const { + return !claim_dir.empty(); + } + // argv[0] is the program name, argv[1] the job file, then option/value pairs. + static Options parse(const std::vector& argv); +}; + +const char* io_name(ReadMode mode); + +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/radio_run.cpp b/src/cuphoton/xbeam/native/cuda/radio_run.cpp new file mode 100644 index 00000000..6c6720f0 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/radio_run.cpp @@ -0,0 +1,757 @@ +// SPDX-License-Identifier: Apache-2.0 +// Native radio execution: no Python or CuPy runtime calls. +#include "gpu.hpp" +#include "radio_model.hpp" +#include "radio_options.hpp" +#include "run_report.hpp" +#include "xbeam/cuda.hpp" +#include "xbeam/execution.hpp" +#include "xbeam/recording.hpp" +#include "xbeam/work_queue.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace xbeam { +namespace { +using gpu::check; +using Clock = std::chrono::steady_clock; +using Extent = + std::pair; // frame range [first, stop) + +class DeviceScope { + int previous_; + +public: + explicit DeviceScope(int device) { + check(cudaGetDevice(&previous_)); + check(cudaSetDevice(device)); + } + DeviceScope(const DeviceScope&) = delete; + DeviceScope& operator=(const DeviceScope&) = delete; + ~DeviceScope() { + cudaSetDevice(previous_); + } +}; +double since(Clock::time_point start) { + return std::chrono::duration(Clock::now() - start).count(); +} + +// One station pipeline: a recorded band of one datastream. +struct Parent { + RadioReplayBatch plan; + StationLayout layout{}; + gpu::Memory parameters; + gpu::Pinned parameters_host; + gpu::Event done; + // Drain station work before releasing parameters on exceptional exits too. + std::unique_ptr executor; +}; +// One physical recording, double-buffered so the next tile reads during compute. +struct Input { + std::unique_ptr recording; + std::unique_ptr delivery[2]; +}; +struct InputTiming { + double index = 0, recording = 0, delivery = 0; +}; + +// A tile of FFT blocks whose reads are in flight. +struct Batch { + std::int64_t first = 0, stop = 0; + int slot = 0; + std::map plans; + std::map extents; + std::vector> deliveries; + // Streams without an accepted block still get a one-frame read. + Extent extent(int stream) const { + auto found = extents.find(stream); + return found == extents.end() ? Extent{0, 1} : found->second; + } +}; + +// Guided self-scheduling: large chunks first, small chunks at the tail. +std::vector guided_chunks( + std::int64_t first, + std::int64_t count, + int workers, + std::int64_t min_chunk) { + std::vector table; + for (std::int64_t position = first, remaining = count; remaining > 0;) { + const auto size = std::min( + remaining, + std::max(min_chunk, (remaining + 2 * workers - 1) / (2 * workers))); + table.emplace_back(position, size); + position += size; + remaining -= size; + } + return table; +} + +// Rethrow the first failure only after every task finished. +template +void join_all(Futures& futures) { + std::exception_ptr failure; + for (auto& future : futures) { + try { + future.get(); + } catch (...) { + if (!failure) { + failure = std::current_exception(); + } + } + } + if (failure) { + std::rethrow_exception(failure); + } +} + +class Run { +public: + explicit Run(Options options); + Json execute(); + +private: + void load_scope(); + void open_inputs(); + void derive_time_grid(); + void allocate_device_state(); + + Batch prepare(std::int64_t first, std::int64_t stop, int slot); + void await_delivery(Batch& batch); + void submit_station( + const Key& key, Parent& parent, const Batch& batch, int rows); + void accumulate_products( + int rows, std::vector& sums, std::vector& weights); + void integrate(std::int64_t index); + void write_records( + std::int64_t index, + std::vector& sums, + const std::vector& weights); + void run_range(std::int64_t first, std::int64_t count); + Json run_chunks(); + Json report(Json chunks) const; + + const Options opts_; + // The clock includes CUDA context creation, as in every earlier report. + const Clock::time_point started_ = Clock::now(); + // Must precede every GPU resource below, which binds to the current device. + const DeviceScope device_scope_{opts_.device}; + const int device_ = opts_.device; + RadioJob native_; + Json job_; + + // Scan geometry. + double integration_ = 0, origin_ = 0, end_ = 0, fft_seconds_ = 0; + std::int64_t width_ = 0, per_integration_ = 0, timeline_ = 0; + std::vector specs_; + std::set keys_; + std::size_t elements_ = 0; // output channels across all products + + // Declared before the workers and GPU state, so they are destroyed last. + std::map inputs_; + std::map parents_; + std::map input_timing_; + + std::ofstream file_; + gpu::Memory output_, weights_, starts_, lengths_, offsets_, weight_offsets_; + WorkQueue workers_; + // Output copies must finish before these pinned destinations are freed. + gpu::Pinned host_output_, host_exposure_; + // Declared before the products so it outlives their destructor. + gpu::Stream xmac_stream_; + CudaProducts products_; + int uploaded_rows_ = -1; + + std::uint64_t records_ = 0; + double job_seconds_ = 0, setup_seconds_ = 0; + double plan_seconds_ = 0, read_seconds_ = 0, front_seconds_ = 0, + xmac_seconds_ = 0; +}; + +Run::Run(Options options) + : opts_(std::move(options)), + native_(opts_.job), + job_(Json::parse(native_.normalized_json())), + workers_(opts_.upload_workers), + products_(opts_.xmac_pack_products, opts_.xmac_reuse_sources) { + load_scope(); + job_seconds_ = since(started_); + open_inputs(); + derive_time_grid(); + allocate_device_state(); + setup_seconds_ = since(started_); +} + +void Run::load_scope() { + const Json* scan = nullptr; + for (const auto& candidate : job_.at("scans")) { + if (candidate.at("identifier") == opts_.scan) { + scan = &candidate; + } + } + if (!scan) { + throw RadioError("unknown execution scan"); + } + auto seconds = [](const Json& value) { + return std::stod(value.get()); + }; + integration_ = seconds(job_.at("integration_duration_s")); + const double subintegration = seconds(job_.at("subintegration_duration_s")), + start_offset = seconds(scan->at("start_offset_s")), + execution = seconds(job_.at("execution_duration_s")), + fractional_start = + seconds(job_.at("start_time").at("seconds_of_day")); + origin_ = std::max(0., start_offset); + end_ = std::min(execution, start_offset + seconds(scan->at("duration_s"))); + end_ = std::min( + end_, execution - (fractional_start - std::floor(fractional_start))); + if (opts_.first_integration < 0 || opts_.integration_count <= 0 + || opts_.fft_tile < 8 || opts_.fft_tile > 1600 || end_ <= origin_ + || integration_ <= 0 || subintegration <= 0) { + throw RadioError("invalid native correlation scope"); + } + specs_ = products(job_); + if (specs_.empty()) { + throw RadioError("job has no visibility products"); + } + for (const auto& p : specs_) { + keys_.insert(p.a.parent); + keys_.insert(p.b.parent); + elements_ += p.channels / p.average; + } +} + +// Each recording's sidecar validation, timeline construction and cuFile +// registration is independent. Build them concurrently, one task per file. +void Run::open_inputs() { + std::vector streams; + for (const auto& key : keys_) { + if (streams.empty() || streams.back() != key.first) { + streams.push_back(key.first); + } + } + for (int stream : streams) { + inputs_[stream]; + input_timing_[stream]; + } + std::vector> tasks; + for (int stream : streams) { + tasks.push_back(std::async(std::launch::async, [this, stream] { + check(cudaSetDevice(device_)); + auto& input = inputs_.at(stream); + auto& timing = input_timing_.at(stream); + const auto& datastream = by_id(job_.at("datastreams"), stream); + if (datastream.at("source_files").size() != 1) { + throw RadioError( + "native replay requires one physical recording"); + } + const std::string source = datastream.at("source_files").at(0); + auto phase = Clock::now(); + auto index = open_index(source, opts_.index_dir); + timing.index = since(phase); + phase = Clock::now(); + input.recording = std::make_unique( + native_, stream, std::move(*index)); + timing.recording = since(phase); + phase = Clock::now(); + const auto& geometry = input.recording->geometry(); + for (int slot = 0; slot < (opts_.prefetch ? 2 : 1); ++slot) { + input.delivery[slot] = std::make_unique( + source, + geometry.frame_bytes, + geometry.payload_bytes, + opts_.read_mode, + opts_.chunk_bytes, + opts_.shared_staging); + } + timing.delivery = since(phase); + })); + } + join_all(tasks); +} + +void Run::derive_time_grid() { + for (const auto& key : keys_) { + const auto& g = inputs_.at(key.first).recording->geometry(); + const double duration = + double(g.request.fft_samples) / double(g.sample_rate); + if (fft_seconds_ && duration != fft_seconds_) { + throw RadioError( + "native execution requires a shared FFT time grid"); + } + fft_seconds_ = duration; + width_ = g.request.ffts_per_request; + parents_.try_emplace(key); + } + if (opts_.fft_tile % width_) { + throw RadioError("FFT tile must contain whole subintegrations"); + } + per_integration_ = std::llround(integration_ / fft_seconds_); + if (std::abs(per_integration_ * fft_seconds_ - integration_) > 1e-12 + || per_integration_ % width_) { + throw RadioError( + "native integration must contain whole reader requests"); + } + const double nominal = std::ceil((end_ - origin_) / fft_seconds_); + if (!std::isfinite(nominal) || nominal < 1 + || nominal >= std::ldexp(1., 63)) { + throw RadioError("native FFT timeline exceeds int64"); + } + timeline_ = static_cast(nominal); + if ((__int128(opts_.first_integration) + opts_.integration_count) + * per_integration_ + > timeline_) { + throw RadioError("selected integrations exceed scan"); + } +} + +void Run::allocate_device_state() { + if (!std::filesystem::create_directory(opts_.output)) { + throw RadioError("native output directory must be new"); + } + if (!opts_.dynamic()) { + file_.open( + std::filesystem::path(opts_.output) / "visibilities.swin", + std::ios::binary | std::ios::out); + if (!file_) { + throw RadioError("cannot create native output"); + } + } + output_.reserve(elements_ * 8); + weights_.reserve(specs_.size() * 8); + starts_.upload(std::vector{0}); + std::vector offsets, weight_offsets; + for (std::size_t i = 0; i < specs_.size(); ++i) { + offsets.push_back(specs_[i].offset); + weight_offsets.push_back(i); + } + offsets_.upload(offsets); + weight_offsets_.upload(weight_offsets); + host_output_.reserve(elements_ * 8); + host_exposure_.reserve(specs_.size() * 8); +} + +// Plan one tile and start reading its frames into delivery slot `slot`. +Batch Run::prepare(std::int64_t first, std::int64_t stop, int slot) { + auto phase = Clock::now(); + Batch batch; + batch.first = first; + batch.stop = stop; + batch.slot = slot; + for (const auto& entry : parents_) { + const auto key = entry.first; + auto& recording = *inputs_.at(key.first).recording; + auto plan = + recording.plan(opts_.scan, key.second, first, stop, timeline_); + const auto& g = recording.geometry(); + for (const auto& r : plan.replay) { + if (r.accepted) { + const auto left = r.physical_start / g.samples_per_frame; + const auto right = std::min( + g.frame_count, + (r.physical_start + g.request.fft_samples + + g.samples_per_frame - 1) + / g.samples_per_frame); + auto [it, inserted] = + batch.extents.try_emplace(key.first, Extent{left, right}); + if (!inserted) { + it->second.first = std::min(it->second.first, left); + it->second.second = std::max(it->second.second, right); + } + } + } + batch.plans.emplace(key, std::move(plan)); + } + plan_seconds_ += since(phase); + for (auto& entry : inputs_) { + auto* input = &entry.second; + const auto range = batch.extent(entry.first); + const auto validity = input->recording->frame_validity( + range.first, range.second - range.first); + for (const auto& chunk : + input->delivery[slot]->begin(range.first, range.second, validity)) { + batch.deliveries.push_back( + workers_.submit([input, slot, chunk, device = device_] { + check(cudaSetDevice(device)); + input->delivery[slot]->read_chunk(chunk); + })); + } + } + return batch; +} + +void Run::await_delivery(Batch& batch) { + const auto phase = Clock::now(); + join_all(batch.deliveries); + for (auto& entry : inputs_) { + entry.second.delivery[batch.slot]->finish(); + entry.second.recording->frame_validity( + batch.extent(entry.first).first, 0); + } + read_seconds_ += since(phase); +} + +// Build the block parameters, (re)create the station executor and enqueue the +// station's FFT pipeline on its own stream. +void Run::submit_station( + const Key& key, Parent& parent, const Batch& batch, int rows) { + auto& input = inputs_.at(key.first); + const auto& g = input.recording->geometry(); + const auto range = batch.extent(key.first); + const auto &datastream = by_id(job_.at("datastreams"), key.first), + &band = by_id(datastream.at("recorded_bands"), key.second), + &frequency = + by_id(job_.at("frequencies"), band.at("frequency_id")); + std::vector parameters; + for (std::size_t k = 0; k < parent.plan.blocks.size(); ++k) { + const auto& b = parent.plan.blocks[k]; + const auto& r = parent.plan.replay[k]; + parameters.push_back( + {b.output_sample, + b.coarse_sample, + b.block_start, + b.delay_zero, + b.delay_step, + b.delay_quadratic, + (double(b.coarse_sample - r.timing_displacement) + / g.sample_rate) + - b.centre_delay, + r.accepted + ? r.physical_start - (b.output_sample - b.coarse_sample) + : 0, + 0, + r.valid_first, + r.valid_stop}); + } + auto& layout = parent.layout; + if (!parent.executor || layout.rows != rows) { + parent.executor.reset(); + layout = {}; + layout.rows = rows; + layout.fft_samples = g.request.fft_samples; + layout.channels = frequency.at("channel_count"); + layout.mapping = frequency.at("sideband") == "USB" ? 0 : 1; + layout.validity_stride = g.samples_per_frame; + layout.samples_per_frame = g.samples_per_frame; + layout.payload_bytes = g.payload_bytes; + layout.channel_count = g.channel_count; + layout.channel_index = parent.plan.channel_index; + layout.sample_rate = g.sample_rate; + layout.reference_hz = frequency.at("band_edge_hz"); + layout.lo_hz = band.at("lo_offset_hz"); + // The declared LO offset uses job-relative block time; there is no + // additional absolute wall-clock phase. + layout.wall_lo_cycles = 0; + layout.source_start = range.first * g.samples_per_frame; + layout.source_samples = + (range.second - range.first) * g.samples_per_frame; + parent.executor = std::make_unique(layout); + } + parent.executor->source_window( + range.first * g.samples_per_frame, + (range.second - range.first) * g.samples_per_frame); + // Stream-ordered upload from pinned memory. The host buffer and device + // parameters are reused only after the tile's stream synchronization. + const auto bytes = parameters.size() * sizeof(BlockParameters); + parent.parameters.reserve(bytes); + parent.parameters_host.reserve(bytes); + std::memcpy(parent.parameters_host.p, parameters.data(), bytes); + check(cudaMemcpyAsync( + parent.parameters.p, + parent.parameters_host.p, + bytes, + cudaMemcpyHostToDevice, + reinterpret_cast(parent.executor->stream()))); + parent.executor->submit( + input.delivery[batch.slot]->payload(), + input.delivery[batch.slot]->validity(), + parent.parameters.address()); +} + +// Cross-multiply the station spectra, download the tile's partial integration +// and add it into `sums` / `weights`. +void Run::accumulate_products( + int rows, std::vector& sums, std::vector& weights) { + if (rows != uploaded_rows_) { + // The previous tile's consumers completed before this point. + lengths_.upload(std::vector{rows}); + uploaded_rows_ = rows; + } + const auto phase = Clock::now(); + std::vector visibility; + for (const auto& p : specs_) { + auto &a = parents_.at(p.a.parent), &b = parents_.at(p.b.parent); + const bool parallel = + p.automatic && p.a.parent == p.b.parent && p.a.start == p.b.start; + const int channels = parallel ? p.channels / p.average : p.channels, + average = parallel ? 1 : p.average; + visibility.push_back( + {a.executor->spectra() + p.a.start * 8, + b.executor->spectra() + p.b.start * 8, + a.executor->weights(), + b.executor->weights(), + a.layout.channels, + b.layout.channels, + channels, + average, + 1. + / std::sqrt( + double(a.layout.fft_samples) * b.layout.fft_samples), + !parallel}); + } + products_.configure(visibility); + products_.submit( + 1, + starts_.address(), + lengths_.address(), + offsets_.address(), + weight_offsets_.address(), + output_.address(), + weights_.address(), + reinterpret_cast(xmac_stream_.value)); + check(cudaMemcpyAsync( + host_output_.p, + output_.p, + elements_ * 8, + cudaMemcpyDeviceToHost, + xmac_stream_.value)); + check(cudaMemcpyAsync( + host_exposure_.p, + weights_.p, + specs_.size() * 8, + cudaMemcpyDeviceToHost, + xmac_stream_.value)); + check(cudaStreamSynchronize(xmac_stream_.value)); + xmac_seconds_ += since(phase); + const float* host = host_output_.as(); + const double* host_weights = host_exposure_.as(); + for (std::size_t k = 0; k < elements_ * 2; ++k) { + sums[k] += host[k]; + } + for (std::size_t k = 0; k < weights.size(); ++k) { + weights[k] += host_weights[k]; + } +} + +// One integration: tiles are read ahead into the second delivery slot while the +// current tile computes. +void Run::integrate(std::int64_t index) { + std::vector sums(elements_ * 2, 0); + std::vector weights(specs_.size(), 0); + const auto first = index * per_integration_, + stop = first + per_integration_; + const auto tile = opts_.fft_tile; + auto current = prepare(first, std::min(stop, first + tile), 0); + while (current.first < stop) { + await_delivery(current); + const int rows = current.stop - current.first; + std::unique_ptr next; + if (opts_.prefetch && current.stop < stop) { + next = std::make_unique(prepare( + current.stop, + std::min(stop, current.stop + tile), + 1 - current.slot)); + } + for (auto& entry : parents_) { + entry.second.plan = std::move(current.plans.at(entry.first)); + } + auto phase = Clock::now(); + for (auto& entry : parents_) { + submit_station(entry.first, entry.second, current, rows); + } + // Order the product stream after every station stream on the GPU, with + // one host wait after the products and output download. A device-wide + // wait would also stall the next-batch delivery streams. + for (auto& entry : parents_) { + auto station = + reinterpret_cast(entry.second.executor->stream()); + check(cudaEventRecord(entry.second.done.value, station)); + check(cudaStreamWaitEvent( + xmac_stream_.value, entry.second.done.value, 0)); + } + front_seconds_ += since(phase); + accumulate_products(rows, sums, weights); + if (current.stop == stop) { + break; + } + current = next + ? std::move(*next) + : prepare(current.stop, std::min(stop, current.stop + tile), 0); + } + write_records(index, sums, weights); + std::cerr << "integration " << index << " records=" << records_ << '\n'; +} + +void Run::write_records( + std::int64_t index, + std::vector& sums, + const std::vector& weights) { + std::map metadata_cache; + for (std::size_t k = 0; k < specs_.size(); ++k) { + if (weights[k] <= 0) { + continue; + } + const auto& p = specs_[k]; + const int channels = p.channels / p.average; + for (int j = 0; j < 2 * channels; ++j) { + sums[p.offset * 2 + j] = + float(double(sums[p.offset * 2 + j]) / weights[k]); + } + const Key stations{p.a.station, p.b.station}; + auto found = metadata_cache.find(stations); + if (found == metadata_cache.end()) { + found = metadata_cache + .emplace( + stations, + native_.record_metadata( + opts_.scan, + index, + p.a.station, + p.b.station, + p.frequency, + p.a.polarization, + p.b.polarization)) + .first; + } + auto metadata = found->second; + metadata.frequency_id = p.frequency; + metadata.polarization_a = p.a.polarization; + metadata.polarization_b = p.b.polarization; + metadata.weight = weights[k] * fft_seconds_ / integration_; + const auto encoded = + encode_swin(metadata, sums.data() + p.offset * 2, channels); + file_.write(encoded.data(), encoded.size()); + if (!file_) { + throw RadioError("native output write failed"); + } + ++records_; + } +} + +void Run::run_range(std::int64_t first, std::int64_t count) { + for (auto index = first; index < first + count; ++index) { + integrate(index); + } +} + +// Dynamic mode: independent worker processes (possibly on several nodes) claim +// chunks of a shared, deterministic table with atomic directory creation, one +// output file per chunk. +Json Run::run_chunks() { + const std::filesystem::path claims(opts_.claim_dir), output(opts_.output); + if (!std::filesystem::is_directory(claims)) { + throw RadioError("claim directory must already exist"); + } + const auto table = guided_chunks( + opts_.first_integration, + opts_.integration_count, + opts_.workers_total, + opts_.min_chunk); + Json reports = Json::array(); + for (std::size_t c = 0; c < table.size(); ++c) { + char claim[64], name[64]; + std::snprintf(claim, sizeof claim, "chunk-%06zu", c); + std::error_code ec; + if (!std::filesystem::create_directory(claims / claim, ec)) { + if (ec) { + throw RadioError("cannot claim chunk: " + ec.message()); + } + continue; // another worker owns it + } + std::snprintf(name, sizeof name, "visibilities.chunk-%06zu.swin", c); + const auto began = Clock::now(); + const auto before = records_; + file_.open(output / name, std::ios::binary | std::ios::out); + if (!file_) { + throw RadioError("cannot create native chunk output"); + } + run_range(table[c].first, table[c].second); + file_.close(); + if (!file_) { + throw RadioError("native chunk close failed"); + } + reports.push_back( + {{"index", c}, + {"first_integration", table[c].first}, + {"integration_count", table[c].second}, + {"records", records_ - before}, + {"seconds", since(began)}, + {"file", std::string(name)}}); + } + return reports; +} + +Json Run::execute() { + Json chunks = Json::array(); + if (opts_.dynamic()) { + chunks = run_chunks(); + } else { + run_range(opts_.first_integration, opts_.integration_count); + file_.close(); + if (!file_) { + throw RadioError("native output close failed"); + } + } + auto result = report(std::move(chunks)); + publish_run_report(opts_.output, result.dump(2)); + return result; +} + +Json Run::report(Json chunks) const { + Json result = {{"backend", "native-cuda"}, + {"profile", opts_.profile}, + {"records", records_}, + {"seconds", since(started_)}, + {"first_integration", opts_.first_integration}, + {"integration_count", opts_.integration_count}, + {"fft_tile", opts_.fft_tile}, + {"io", io_name(opts_.read_mode)}}; + result["setup_seconds"] = setup_seconds_; + result["dynamic"] = opts_.dynamic(); + result["chunks"] = std::move(chunks); + result["upload_workers"] = opts_.upload_workers; + result["read_chunk_bytes"] = opts_.chunk_bytes; + result["read_staging"] = opts_.shared_staging ? "shared" : "per-input"; + result["xmac_pack_products"] = opts_.xmac_pack_products; + result["xmac_reuse_sources"] = opts_.xmac_reuse_sources; + result["prefetch"] = opts_.prefetch; + result["read_measurement"] = + "exposed delivery wait; prefetched IO can overlap compute"; + result["setup_job_seconds"] = job_seconds_; + Json setup_inputs = Json::object(); + for (const auto& [stream, timing] : input_timing_) { + setup_inputs[std::to_string(stream)] = { + {"index_validation_seconds", timing.index}, + {"recording_seconds", timing.recording}, + {"delivery_open_seconds", timing.delivery}}; + } + result["setup_inputs"] = setup_inputs; + result["timing_note"] = + "front_seconds is host submission; xmac_seconds includes the GPU wait"; + result["plan_seconds"] = plan_seconds_; + result["read_seconds"] = read_seconds_; + result["front_seconds"] = front_seconds_; + result["xmac_seconds"] = xmac_seconds_; + return result; +} +} // namespace + +std::string correlate_radio(const std::vector& arguments) { + for (const auto& argument : arguments) { + if (argument.find('\0') != std::string::npos) { + throw RadioError("native execution argument contains NUL"); + } + } + return Run(Options::parse(arguments)).execute().dump(); +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/run_report.hpp b/src/cuphoton/xbeam/native/cuda/run_report.hpp new file mode 100644 index 00000000..9ecc45e0 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/run_report.hpp @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "xbeam/radio.hpp" +#include +#include +#include +namespace xbeam { + +// A completion marker becomes visible only after its entire contents close. +inline void publish_run_report( + const std::filesystem::path& directory, const std::string& contents) { + const auto target = directory / "native-run.json"; + const auto temporary = directory / "native-run.json.tmp"; + try { + std::ofstream report(temporary, std::ios::binary | std::ios::out); + if (!report) { + throw RadioError("cannot create native completion report"); + } + report << contents << '\n'; + report.close(); + if (!report) { + throw RadioError("native completion report write failed"); + } + std::error_code error; + std::filesystem::rename(temporary, target, error); + if (error) { + throw RadioError( + "cannot publish native completion report: " + error.message()); + } + } catch (...) { + std::error_code ignored; + std::filesystem::remove(temporary, ignored); + throw; + } +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/series.cpp b/src/cuphoton/xbeam/native/cuda/series.cpp new file mode 100644 index 00000000..c1669d17 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/series.cpp @@ -0,0 +1,678 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/series.hpp" +#include "xbeam/work_queue.hpp" +#include "gpu.hpp" +#include +#include +#ifdef XBEAM_HAVE_CUFILE +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using xbeam::gpu::check; +namespace xbeam { +namespace { +void fft_check(cufftResult result) { + if (result != CUFFT_SUCCESS) { + throw RadioError( + "native series cuFFT failure: " + std::to_string(result)); + } +} +std::size_t bounded_bytes(std::int64_t count, std::size_t width) { + if (count < 0 + || static_cast(count) > (1ULL << 30) / width) { + throw RadioError( + "native series buffer exceeds 1 GiB; reduce batch_windows"); + } + return std::size_t(count) * width; +} +struct File { + int fd = -1; + struct stat snapshot{}; + bool direct = false; +#ifdef XBEAM_HAVE_CUFILE + CUfileHandle_t handle = nullptr; +#endif + ~File() { +#ifdef XBEAM_HAVE_CUFILE + if (handle) { + cuFileHandleDeregister(handle); + } +#endif + if (fd >= 0) { + ::close(fd); + } + } + void open(const nlohmann::json& config) { + const std::string mode = config.at("io"); + if (mode != "posix" && mode != "cufile") { + throw RadioError("unknown native series I/O mode"); + } + direct = mode == "cufile"; + const std::string path = config.at("path"); + if (path.find('\0') != std::string::npos) { + throw RadioError("NUL in native series path"); + } + fd = ::open( + path.c_str(), O_RDONLY | O_CLOEXEC | (direct ? O_DIRECT : 0)); + if (fd < 0 || ::fstat(fd, &snapshot) || !S_ISREG(snapshot.st_mode)) { + throw RadioError("cannot open native series input"); + } + if (std::uint64_t(snapshot.st_dev) + != config.at("device").get() + || std::uint64_t(snapshot.st_ino) + != config.at("inode").get()) { + throw RadioError("HDF5 source changed before native ingest"); + } + if (direct) { +#ifdef XBEAM_HAVE_CUFILE + ensure_cufile_driver(); + CUfileDescr_t descriptor{}; + descriptor.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD; + descriptor.handle.fd = fd; + if (cuFileHandleRegister(&handle, &descriptor).err + != CU_FILE_SUCCESS) { + throw RadioError( + "native series cuFile handle registration failed"); + } +#else + throw RadioError("native CUDA build does not include cuFile"); +#endif + } + } + void unchanged() const { + struct stat now{}; + if (::fstat(fd, &now) || now.st_size != snapshot.st_size + || now.st_mtim.tv_sec != snapshot.st_mtim.tv_sec + || now.st_mtim.tv_nsec != snapshot.st_mtim.tv_nsec) { + throw RadioError("native series input changed during ingest"); + } + } +}; +struct Raw { + gpu::Memory device{65536}; + gpu::Pinned host; + std::size_t prefix = 0, size = 0; +#ifdef XBEAM_HAVE_CUFILE + void* registered = nullptr; +#endif + ~Raw() { +#ifdef XBEAM_HAVE_CUFILE + if (registered) { + cuFileBufDeregister(registered); + } +#endif + } + void read(const File& file, std::int64_t start, std::size_t bytes) { + if (start < 0 || bytes > std::size_t(file.snapshot.st_size) + || std::uint64_t(start) + > std::uint64_t(file.snapshot.st_size) - bytes) { + throw RadioError("native series read outside file extent"); + } + const auto begin = file.direct ? start / 65536 * 65536 : start; + prefix = start - begin; + size = bytes; + const auto count = file.direct + ? std::min( + file.snapshot.st_size - begin, + (prefix + bytes + 65535) / 65536 * 65536) + : bytes; + if (count <= 0 || count > (1LL << 30)) { + throw RadioError("native series read exceeds bounded extent"); + } +#ifdef XBEAM_HAVE_CUFILE + if (file.direct && count > device.capacity && registered) { + if (cuFileBufDeregister(registered).err != CU_FILE_SUCCESS) { + throw RadioError("native series cuFile deregistration failed"); + } + registered = nullptr; + } +#endif + device.reserve(count); + if (file.direct) { +#ifdef XBEAM_HAVE_CUFILE + if (!registered) { + if (cuFileBufRegister(device.p, device.capacity, 0).err + != CU_FILE_SUCCESS) { + throw RadioError( + "native series cuFile buffer registration failed"); + } + registered = device.p; + } + const auto got = cuFileRead(file.handle, device.p, count, begin, 0); + if (got < 0 || std::size_t(got) < prefix + bytes) { + throw RadioError("short native series cuFile read"); + } +#endif + } else { + host.reserve(count); + std::size_t done = 0; + while (done < std::size_t(count)) { + const auto got = ::pread( + file.fd, + static_cast(host.p) + done, + count - done, + begin + done); + if (got < 0 && errno == EINTR) { + continue; + } + if (got <= 0) { + throw RadioError("short native series POSIX read"); + } + done += got; + } + } + } + const void* upload(const File& file, cudaStream_t stream) { + if (!file.direct) { + check(cudaMemcpyAsync( + device.p, host.p, size, cudaMemcpyHostToDevice, stream)); + } + return static_cast(device.p) + prefix; + } +}; +struct Input { + std::int64_t offset, mask_offset; + int dtype, itemsize; + Raw raw[2], masks[2]; + gpu::Memory spectrum, mask_spectrum, validity; +}; +struct DeviceSelection { + int device, previous; + explicit DeviceSelection(const std::string& text) { + check(cudaGetDevice(&previous)); + device = nlohmann::json::parse(text).at("gpu").get(); + if (device == -1) { + device = previous; + } + check(cudaSetDevice(device)); + } + ~DeviceSelection() { + cudaSetDevice(previous); + } +}; +struct DeviceScope { + int previous; + explicit DeviceScope(int device) { + check(cudaGetDevice(&previous)); + check(cudaSetDevice(device)); + } + ~DeviceScope() { + cudaSetDevice(previous); + } +}; +struct Plans { + cufftHandle samples = 0, masks = 0; + ~Plans() { + if (samples) { + cufftDestroy(samples); + } + if (masks) { + cufftDestroy(masks); + } + } +}; +} // namespace +struct NativeSeries::Impl { + DeviceSelection selected; + gpu::Stream stream; + File file; + int device, n, fft_n, group_n, batch_n, pair_count, lag_count, lag_first; + std::int64_t sample_count, hop; + bool wide, want_lag, want_spectrum, prefetch, masks = false; + std::size_t complex_bytes; + std::vector> inputs; + std::vector> pairs; + std::map> plans; + std::unique_ptr workers; + gpu::Event consumed[2]; + bool recorded[2]{false, false}; + gpu::Memory product, total, mask_product, mask_total, exposure, + dummy_weights, inverse, mask_inverse, lag, coverage, statistics, error; + Plans inverse_plans; + std::mutex active; + explicit Impl(const std::string& text) + : selected(text) { + const auto c = nlohmann::json::parse(text); + device = selected.device; + check(cudaSetDevice(device)); + n = c.at("window_samples"); + fft_n = c.at("fft_samples"); + group_n = c.at("windows_per_result"); + batch_n = c.at("batch_windows"); + sample_count = c.at("sample_count"); + hop = c.at("hop_samples"); + wide = c.at("wide"); + prefetch = c.at("prefetch"); + want_lag = c.at("want_lag"); + want_spectrum = c.at("want_spectrum"); + const bool linear = c.at("linear"); + if (n <= 0 || n > (1 << 26) || fft_n < n || fft_n > (1 << 27) + || group_n <= 0 || batch_n <= 0 || batch_n > 65535 || hop <= 0 + || sample_count < 0 || (!want_lag && !want_spectrum) + || (linear && std::int64_t(fft_n) < 2LL * n - 1) + || (!linear && fft_n != n)) { + throw RadioError("invalid native series geometry"); + } + batch_n = std::min(batch_n, group_n); + const std::int64_t read_limit = c.at("read_buffer_bytes"); + if (read_limit < 4096 || read_limit > (1LL << 30)) { + throw RadioError("invalid native read buffer size"); + } + for (const auto& input : c.at("inputs")) { + const int dtype = input.at("dtype"); + const int itemsize = dtype == 0 ? 4 : dtype == 3 ? 16 : 8; + const auto maximum = read_limit / itemsize; + if (maximum < n) { + throw RadioError("native window exceeds read_buffer_bytes"); + } + batch_n = + int(std::min(batch_n, 1 + (maximum - n) / hop)); + } + lag_first = linear ? -(n - 1) : -(n / 2); + lag_count = linear ? 2 * n - 1 : n; + complex_bytes = wide ? 16 : 8; + const auto sample_bytes = + bounded_bytes(std::int64_t(batch_n) * fft_n, complex_bytes); + file.open(c); + for (const auto& entry : c.at("inputs")) { + auto input = std::make_unique(); + input->offset = entry.at("offset"); + input->mask_offset = entry.at("mask_offset"); + input->dtype = entry.at("dtype"); + if (input->dtype < 0 || input->dtype > 3 + || input->mask_offset < -1) { + throw RadioError( + "invalid native series dtype or validity extent"); + } + input->itemsize = input->dtype == 0 ? 4 + : input->dtype == 3 ? 16 + : 8; + for (auto range : {std::make_pair(input->offset, input->itemsize), + std::make_pair(input->mask_offset, 1)}) { + if (range.first == -1) { + continue; + } + if (range.first < 0 + || __int128(range.first) + + __int128(sample_count) * range.second + > file.snapshot.st_size) { + throw RadioError( + "native series dataset exceeds input file"); + } + } + input->spectrum.reserve(sample_bytes); + input->validity.reserve( + bounded_bytes(std::int64_t(batch_n) * n, 1)); + masks |= input->mask_offset >= 0; + inputs.push_back(std::move(input)); + } + if (inputs.empty()) { + throw RadioError("native series needs input signals"); + } + for (const auto& pair : c.at("pairs")) { + if (pair.size() != 2) { + throw RadioError("invalid native series pair"); + } + const int a = pair.at(0), b = pair.at(1); + if (a < 0 || b < 0 || a >= int(inputs.size()) + || b >= int(inputs.size())) { + throw RadioError("unknown native series pair input"); + } + pairs.emplace_back(a, b); + } + if (pairs.empty() || pairs.size() > 65535) { + throw RadioError("invalid native series pair count"); + } + pair_count = pairs.size(); + const auto total_bytes = + bounded_bytes(std::int64_t(pair_count) * fft_n, complex_bytes); + total.reserve(total_bytes); + product.reserve(std::size_t(fft_n) * complex_bytes); + exposure.reserve(8); + error.reserve(4); + statistics.reserve(pair_count * 2 * 8); + dummy_weights.upload(std::vector(batch_n, 1)); + const int threads = c.at("upload_workers"); + if (threads < 1 || threads > 64) { + throw RadioError("native series upload_workers must be in [1,64]"); + } + workers = std::make_unique(threads); + if (want_lag) { + inverse.reserve(total_bytes); + lag.reserve(bounded_bytes( + std::int64_t(pair_count) * lag_count, complex_bytes)); + coverage.reserve( + bounded_bytes(std::int64_t(pair_count) * lag_count, 8)); + fft_check(cufftPlan1d( + &inverse_plans.samples, + fft_n, + wide ? CUFFT_Z2Z : CUFFT_C2C, + pair_count)); + fft_check(cufftSetStream(inverse_plans.samples, stream.value)); + if (masks) { + const auto mask_bytes = + bounded_bytes(std::int64_t(pair_count) * fft_n, 16); + mask_total.reserve(mask_bytes); + mask_inverse.reserve(mask_bytes); + mask_product.reserve(std::size_t(fft_n) * 16); + for (auto& input : inputs) { + input->mask_spectrum.reserve( + bounded_bytes(std::int64_t(batch_n) * fft_n, 16)); + } + fft_check(cufftPlan1d( + &inverse_plans.masks, fft_n, CUFFT_Z2Z, pair_count)); + fft_check(cufftSetStream(inverse_plans.masks, stream.value)); + } + } + check(cudaSetDevice(selected.previous)); + } + ~Impl() { + cudaGetDevice(&selected.previous); + cudaSetDevice(device); + cudaStreamSynchronize(stream.value); + } + Plans& plan(int rows) { + auto& p = plans[rows]; + if (!p) { + p = std::make_unique(); + fft_check(cufftPlan1d( + &p->samples, fft_n, wide ? CUFFT_Z2Z : CUFFT_C2C, rows)); + fft_check(cufftSetStream(p->samples, stream.value)); + if (want_lag && masks) { + fft_check(cufftPlan1d(&p->masks, fft_n, CUFFT_Z2Z, rows)); + fft_check(cufftSetStream(p->masks, stream.value)); + } + } + return *p; + } + void load(int slot, std::int64_t first, int rows) { + check(cudaSetDevice(device)); + if (recorded[slot]) { + check(cudaEventSynchronize(consumed[slot].value)); + } + const std::int64_t count = std::int64_t(rows - 1) * hop + n; + std::vector> pending; + for (auto& owner : inputs) { + auto* input = owner.get(); + pending.push_back( + workers->submit([this, input, slot, first, count] { + check(cudaSetDevice(device)); + input->raw[slot].read( + file, + input->offset + first * input->itemsize, + bounded_bytes(count, input->itemsize)); + if (input->mask_offset >= 0) { + input->masks[slot].read( + file, + input->mask_offset + first, + bounded_bytes(count, 1)); + } + })); + } + std::exception_ptr failure; + for (auto& task : pending) { + try { + task.get(); + } catch (...) { + if (!failure) { + failure = std::current_exception(); + } + } + } + if (failure) { + std::rethrow_exception(failure); + } + } + void compute(int slot, int rows) { + auto& p = plan(rows); + const auto work = reinterpret_cast(stream.value); + for (auto& input : inputs) { + const auto* raw = input->raw[slot].upload(file, stream.value); + const auto* mask = input->mask_offset < 0 + ? nullptr + : static_cast( + input->masks[slot].upload(file, stream.value)); + pack_series_input( + raw, + mask, + input->dtype, + wide, + rows, + n, + fft_n, + hop, + input->spectrum.p, + static_cast(input->validity.p), + want_lag && masks ? input->mask_spectrum.p : nullptr, + static_cast(error.p), + work); + if (wide) { + fft_check(cufftExecZ2Z( + p.samples, + static_cast(input->spectrum.p), + static_cast(input->spectrum.p), + CUFFT_FORWARD)); + } else { + fft_check(cufftExecC2C( + p.samples, + static_cast(input->spectrum.p), + static_cast(input->spectrum.p), + CUFFT_FORWARD)); + } + if (want_lag && masks) { + fft_check(cufftExecZ2Z( + p.masks, + static_cast(input->mask_spectrum.p), + static_cast(input->mask_spectrum.p), + CUFFT_FORWARD)); + } + } + for (int k = 0; k < pair_count; ++k) { + auto& a = *inputs[pairs[k].first]; + auto& b = *inputs[pairs[k].second]; + sum_series_spectra( + wide, + rows, + fft_n, + a.spectrum.p, + b.spectrum.p, + static_cast(dummy_weights.p), + 1. / group_n, + product.p, + static_cast(exposure.p), + work); + add_series_spectrum( + wide, + fft_n, + product.p, + static_cast(total.p) + + std::size_t(k) * fft_n * complex_bytes, + work); + series_statistics( + static_cast(a.validity.p), + static_cast(b.validity.p), + rows, + n, + static_cast(statistics.p) + 2 * k, + work); + if (want_lag && masks) { + sum_series_spectra( + true, + rows, + fft_n, + a.mask_spectrum.p, + b.mask_spectrum.p, + static_cast(dummy_weights.p), + 1, + mask_product.p, + static_cast(exposure.p), + work); + add_series_spectrum( + true, + fft_n, + mask_product.p, + static_cast(mask_total.p) + + std::size_t(k) * fft_n * 16, + work); + } + } + check(cudaEventRecord(consumed[slot].value, stream.value)); + recorded[slot] = true; + } +}; +NativeSeries::NativeSeries(const std::string& c) + : impl(std::make_unique(c)) {} +NativeSeries::~NativeSeries() = default; +void NativeSeries::run( + std::int64_t first, + int windows, + void* spectrum, + std::size_t spectrum_bytes, + void* lag, + std::size_t lag_bytes, + std::int64_t* statistics, + std::size_t statistics_count, + std::int64_t* coverage, + std::size_t coverage_count) { + auto& p = *impl; + std::unique_lock lock(p.active, std::try_to_lock); + if (!lock.owns_lock()) { + throw RadioError("native series executor is already active"); + } + DeviceScope selected(p.device); + if (windows != p.group_n || first < 0 + || __int128(first) + __int128(windows - 1) * p.hop + p.n + > p.sample_count + || spectrum_bytes + != (p.want_spectrum + ? std::size_t(p.pair_count) * p.fft_n * p.complex_bytes + : 0) + || lag_bytes + != (p.want_lag + ? std::size_t(p.pair_count) * p.lag_count * p.complex_bytes + : 0) + || statistics_count != std::size_t(p.pair_count) * 2 + || coverage_count + != (p.want_lag ? std::size_t(p.pair_count) * p.lag_count : 0) + || !statistics || (spectrum_bytes && !spectrum) || (lag_bytes && !lag) + || (coverage_count && !coverage)) { + throw RadioError( + "invalid native series result scope or output buffers"); + } + p.file.unchanged(); + check(cudaMemsetAsync( + p.total.p, + 0, + std::size_t(p.pair_count) * p.fft_n * p.complex_bytes, + p.stream.value)); + check(cudaMemsetAsync( + p.statistics.p, 0, p.pair_count * 2 * 8, p.stream.value)); + check(cudaMemsetAsync(p.error.p, 0, 4, p.stream.value)); + if (p.want_lag && p.masks) { + check(cudaMemsetAsync( + p.mask_total.p, + 0, + std::size_t(p.pair_count) * p.fft_n * 16, + p.stream.value)); + } + int offset = 0, slot = 0, rows = std::min(p.batch_n, windows); + p.load(slot, first, rows); + while (offset < windows) { + p.compute(slot, rows); + if (!p.prefetch) { + check(cudaStreamSynchronize(p.stream.value)); + } + offset += rows; + if (offset == windows) { + break; + } + slot = 1 - slot; + rows = std::min(p.batch_n, windows - offset); + // The transfer threads can read the next slot while queued GPU work runs. + p.load(slot, first + std::int64_t(offset) * p.hop, rows); + } + int error = 0; + check(cudaMemcpyAsync( + &error, p.error.p, 4, cudaMemcpyDeviceToHost, p.stream.value)); + check(cudaStreamSynchronize(p.stream.value)); + if (error) { + throw RadioError( + error & 2 ? "invalid Boolean native series validity" + : "nonfinite valid native series sample"); + } + if (spectrum_bytes) { + check(cudaMemcpy( + spectrum, p.total.p, spectrum_bytes, cudaMemcpyDeviceToHost)); + } + check(cudaMemcpy( + statistics, + p.statistics.p, + statistics_count * 8, + cudaMemcpyDeviceToHost)); + if (p.want_lag) { + if (p.wide) { + fft_check(cufftExecZ2Z( + p.inverse_plans.samples, + static_cast(p.total.p), + static_cast(p.inverse.p), + CUFFT_INVERSE)); + } else { + fft_check(cufftExecC2C( + p.inverse_plans.samples, + static_cast(p.total.p), + static_cast(p.inverse.p), + CUFFT_INVERSE)); + } + if (p.masks) { + fft_check(cufftExecZ2Z( + p.inverse_plans.masks, + static_cast(p.mask_total.p), + static_cast(p.mask_inverse.p), + CUFFT_INVERSE)); + } + gather_series_lag( + p.wide, + p.pair_count, + p.fft_n, + p.lag_first, + p.lag_count, + p.inverse.p, + p.lag.p, + p.masks ? p.mask_inverse.p : nullptr, + static_cast(p.coverage.p), + reinterpret_cast(p.stream.value)); + check(cudaMemcpyAsync( + lag, p.lag.p, lag_bytes, cudaMemcpyDeviceToHost, p.stream.value)); + if (p.masks) { + check(cudaMemcpyAsync( + coverage, + p.coverage.p, + coverage_count * 8, + cudaMemcpyDeviceToHost, + p.stream.value)); + } + check(cudaStreamSynchronize(p.stream.value)); + if (!p.masks) { + for (int k = 0; k < p.pair_count; ++k) { + for (int j = 0; j < p.lag_count; ++j) { + coverage[std::size_t(k) * p.lag_count + j] = + std::int64_t(windows) + * (p.lag_count == p.n + ? p.n + : p.n - std::abs(p.lag_first + j)); + } + } + } + } + p.file.unchanged(); +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/series_kernels.cu b/src/cuphoton/xbeam/native/cuda/series_kernels.cu new file mode 100644 index 00000000..e1ad7b03 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/series_kernels.cu @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/series.hpp" +#include "gpu.hpp" +#include +namespace xbeam { +namespace { +template +__global__ void pack( + const unsigned char* raw, + const unsigned char* mask, + int dtype, + int rows, + int samples, + int fft_n, + long long hop, + Complex* out, + unsigned char* valid, + double2* masks, + int* error) { + for (long long i = (long long)blockIdx.x * blockDim.x + threadIdx.x; + i < (long long)rows * fft_n; + i += (long long)gridDim.x * blockDim.x) { + const int row = i / fft_n, col = i % fft_n; + double re = 0, im = 0; + bool live = false; + if (col < samples) { + const long long source = row * hop + col; + const auto flag = mask ? mask[source] : 1; + if (flag > 1) { + atomicOr(error, 2); + } + live = flag == 1; + if (live) { + if (dtype == 0) { + re = reinterpret_cast(raw)[source]; + } else if (dtype == 1) { + re = reinterpret_cast(raw)[source]; + } else if (dtype == 2) { + const auto value = + reinterpret_cast(raw)[source]; + re = value.x; + im = value.y; + } else { + const auto value = + reinterpret_cast(raw)[source]; + re = value.x; + im = value.y; + } + if (!isfinite(re) || !isfinite(im)) { + atomicOr(error, 1); + } + } + valid[(long long)row * samples + col] = live; + } + Complex value; + value.x = re; + value.y = im; + if (live + && (!isfinite(double(value.x)) || !isfinite(double(value.y)))) { + atomicOr(error, 1); + } + out[i] = value; + if (masks) { + masks[i] = make_double2(live ? 1 : 0, 0); + } + } +} +__global__ void stats( + const unsigned char* a, + const unsigned char* b, + int rows, + int samples, + unsigned long long* out) { + const int row = blockIdx.x; + unsigned long long overlap = 0, first = 0, second = 0; + for (int i = threadIdx.x; i < samples; i += blockDim.x) { + const auto offset = (long long)row * samples + i; + first += a[offset] != 0; + second += b[offset] != 0; + overlap += a[offset] && b[offset]; + } + __shared__ unsigned long long values[3][256]; + values[0][threadIdx.x] = overlap; + values[1][threadIdx.x] = first; + values[2][threadIdx.x] = second; + __syncthreads(); + for (int stride = 128; stride; stride /= 2) { + if (threadIdx.x < stride) { + for (int k = 0; k < 3; ++k) { + values[k][threadIdx.x] += values[k][threadIdx.x + stride]; + } + } + __syncthreads(); + } + if (!threadIdx.x) { + atomicAdd(out, (unsigned long long)(values[1][0] && values[2][0])); + atomicAdd(out + 1, values[0][0]); + } +} +template +__global__ void add(int n, const Complex* a, Complex* b) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; + i += gridDim.x * blockDim.x) { + b[i].x += a[i].x; + b[i].y += a[i].y; + } +} +template +__global__ void gather( + int pairs, + int fft_n, + int first, + int count, + const Complex* src, + Complex* dst, + const double2* masks, + long long* coverage) { + for (long long i = (long long)blockIdx.x * blockDim.x + threadIdx.x; + i < (long long)pairs * count; + i += (long long)gridDim.x * blockDim.x) { + const int lag = first + i % count; + const int bin = ((-lag) % fft_n + fft_n) % fft_n; + const auto index = (i / count) * fft_n + bin; + Complex value; + value.x = src[index].x / fft_n; + value.y = src[index].y / fft_n; + dst[i] = value; + if (masks) { + coverage[i] = llround(masks[index].x / fft_n); + } + } +} +} // namespace +void pack_series_input( + const void* raw, + const unsigned char* mask, + int dtype, + bool wide, + int rows, + int samples, + int fft_n, + std::int64_t hop, + void* output, + unsigned char* validity, + void* masks, + int* error, + std::uintptr_t stream) { + const auto s = reinterpret_cast(stream); + if (wide) { + pack<<<256, 256, 0, s>>>( + static_cast(raw), + mask, + dtype, + rows, + samples, + fft_n, + hop, + static_cast(output), + validity, + static_cast(masks), + error); + } else { + pack<<<256, 256, 0, s>>>( + static_cast(raw), + mask, + dtype, + rows, + samples, + fft_n, + hop, + static_cast(output), + validity, + static_cast(masks), + error); + } + gpu::check(cudaGetLastError()); +} +void series_statistics( + const unsigned char* a, + const unsigned char* b, + int rows, + int samples, + std::int64_t* out, + std::uintptr_t stream) { + stats<<(stream)>>>( + a, b, rows, samples, reinterpret_cast(out)); + gpu::check(cudaGetLastError()); +} +void add_series_spectrum( + bool wide, + int channels, + const void* input, + void* output, + std::uintptr_t stream) { + const auto s = reinterpret_cast(stream); + if (wide) { + add<<<256, 256, 0, s>>>( + channels, + static_cast(input), + static_cast(output)); + } else { + add<<<256, 256, 0, s>>>( + channels, + static_cast(input), + static_cast(output)); + } + gpu::check(cudaGetLastError()); +} +void gather_series_lag( + bool wide, + int pairs, + int fft_n, + int first, + int count, + const void* inverse, + void* out, + const void* masks, + std::int64_t* coverage, + std::uintptr_t stream) { + const auto s = reinterpret_cast(stream); + if (wide) { + gather<<<256, 256, 0, s>>>( + pairs, + fft_n, + first, + count, + static_cast(inverse), + static_cast(out), + static_cast(masks), + reinterpret_cast(coverage)); + } else { + gather<<<256, 256, 0, s>>>( + pairs, + fft_n, + first, + count, + static_cast(inverse), + static_cast(out), + static_cast(masks), + reinterpret_cast(coverage)); + } + gpu::check(cudaGetLastError()); +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/station.cpp b/src/cuphoton/xbeam/native/cuda/station.cpp new file mode 100644 index 00000000..7a22af76 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/station.cpp @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/cuda.hpp" +#include "xbeam/radio.hpp" +#include +#include +#include +#include +namespace xbeam { +void launch_front( + const StationLayout&, + const void*, + const void*, + const void*, + void*, + void*, + void*, + cudaStream_t); +void launch_channel( + const StationLayout&, const void*, const void*, void*, cudaStream_t); +namespace { +void check(cudaError_t e) { + if (e != cudaSuccess) { + throw RadioError(cudaGetErrorString(e)); + } +} +void check_fft(cufftResult e) { + if (e != CUFFT_SUCCESS) { + throw RadioError("cuFFT error " + std::to_string(int(e))); + } +} +struct Memory { + void* pointer = nullptr; + void allocate(std::size_t bytes) { + if (bytes) { + check(cudaMalloc(&pointer, bytes)); + } + } + ~Memory() { + if (pointer) { + cudaFree(pointer); + } + } +}; +struct Stream { + cudaStream_t value = nullptr; + Stream() { + check(cudaStreamCreateWithFlags(&value, cudaStreamNonBlocking)); + } + ~Stream() { + if (value) { + cudaStreamSynchronize(value); + cudaStreamDestroy(value); + } + } +}; +struct FFT { + cufftHandle value = 0; + FFT() { + check_fft(cufftCreate(&value)); + } + ~FFT() { + if (value) { + cufftDestroy(value); + } + } +}; +std::size_t size(int rows, int columns, std::size_t item) { + if (rows <= 0 || columns <= 0 + || std::size_t(rows) + > std::numeric_limits::max() / columns / item) { + throw RadioError("invalid native CUDA shape"); + } + return std::size_t(rows) * columns * item; +} +} // namespace +struct CudaStation::Impl { + StationLayout g; + // Stream destroyed before buffers: all queued consumers must finish first. + Memory rotated, output, weight, counts, workspace; + FFT plan; + Stream work; + std::size_t output_bytes, weight_bytes; + explicit Impl(const StationLayout& layout) + : g(layout) { + if (g.channels <= 0 || g.channels > g.fft_samples || g.mapping < 0 + || g.mapping > 3 || (g.mapping >= 2 && g.channels != g.fft_samples) + || !std::isfinite(g.sample_rate) || g.sample_rate <= 0 + || !std::isfinite(g.reference_hz) || !std::isfinite(g.lo_hz) + || !std::isfinite(g.wall_lo_cycles) || g.validity_stride < 1 + || g.samples_per_frame < 1 || g.payload_bytes < 1 + || g.channel_count < 1 || g.channel_index < 0 + || g.channel_index >= g.channel_count || g.source_samples < 1) { + throw RadioError("invalid native CUDA station geometry"); + } + const auto bits = std::int64_t(g.samples_per_frame) * g.channel_count + * (g.complex_input ? 4 : 2); + if (bits != std::int64_t(g.payload_bytes) * 8 + || g.source_samples % g.samples_per_frame) { + throw RadioError("inconsistent native packed sample geometry"); + } + output_bytes = size(g.rows, g.channels, 8); + weight_bytes = size(g.rows, 1, 8); + rotated.allocate(size(g.rows, g.fft_samples, 8)); + output.allocate(output_bytes); + weight.allocate(weight_bytes); + if (!g.complex_input && g.fft_samples == 262144 && g.rows >= 8 + && g.rows <= 1600) { + counts.allocate(size(g.rows, 64, 4)); + } + check_fft(cufftSetAutoAllocation(plan.value, 0)); + std::size_t bytes = 0; + check_fft(cufftMakePlan1d( + plan.value, g.fft_samples, CUFFT_C2C, g.rows, &bytes)); + workspace.allocate(bytes); + check_fft(cufftSetWorkArea(plan.value, workspace.pointer)); + check_fft(cufftSetStream(plan.value, work.value)); + } +}; +CudaStation::CudaStation(const StationLayout& g) + : impl(std::make_unique(g)) {} +CudaStation::~CudaStation() = default; +void CudaStation::submit( + std::uintptr_t source, std::uintptr_t validity, std::uintptr_t parameters) { + if (!source || !validity || !parameters) { + throw RadioError("missing native CUDA input address"); + } + auto& p = *impl; + launch_front( + p.g, + reinterpret_cast(source), + reinterpret_cast(validity), + reinterpret_cast(parameters), + p.rotated.pointer, + p.weight.pointer, + p.counts.pointer, + p.work.value); + check(cudaGetLastError()); + check_fft(cufftExecC2C( + p.plan.value, + static_cast(p.rotated.pointer), + static_cast(p.rotated.pointer), + CUFFT_FORWARD)); + launch_channel( + p.g, + p.rotated.pointer, + reinterpret_cast(parameters), + p.output.pointer, + p.work.value); + check(cudaGetLastError()); +} +void CudaStation::download(float* spectra, double* weights) { + if (!spectra || !weights) { + throw RadioError("missing native output buffers"); + } + check(cudaMemcpyAsync( + spectra, + impl->output.pointer, + impl->output_bytes, + cudaMemcpyDeviceToHost, + impl->work.value)); + check(cudaMemcpyAsync( + weights, + impl->weight.pointer, + impl->weight_bytes, + cudaMemcpyDeviceToHost, + impl->work.value)); + check(cudaStreamSynchronize(impl->work.value)); +} +std::uintptr_t CudaStation::stream() const { + return reinterpret_cast(impl->work.value); +} +std::uintptr_t CudaStation::spectra() const { + return reinterpret_cast(impl->output.pointer); +} +std::uintptr_t CudaStation::weights() const { + return reinterpret_cast(impl->weight.pointer); +} +} // namespace xbeam + +namespace xbeam { +void CudaStation::source_window(std::int64_t first, std::int64_t count) { + if (count < 1 || count % impl->g.samples_per_frame) { + throw RadioError("invalid packed source window"); + } + impl->g.source_start = first; + impl->g.source_samples = count; +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/cuda/station_file_test.cpp b/src/cuphoton/xbeam/native/cuda/station_file_test.cpp new file mode 100644 index 00000000..4de8b827 --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/station_file_test.cpp @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: Apache-2.0 +// Bounded real-recording frontend check. This is not a correlator. +#include "xbeam/cuda.hpp" +#include "xbeam/radio.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +namespace { +using J = nlohmann::json; +void check(cudaError_t c) { + if (c != cudaSuccess) { + throw std::runtime_error(cudaGetErrorString(c)); + } +} +struct Buffer { + void* p = nullptr; + explicit Buffer(std::size_t n) { + check(cudaMalloc(&p, n)); + } + ~Buffer() { + if (p) { + cudaFree(p); + } + } +}; +const J& id(const J& a, int i) { + for (const auto& x : a) { + if (x.at("identifier") == i) { + return x; + } + } + throw std::runtime_error("unknown identifier"); +} +} // namespace +int main(int argc, char** argv) { + try { + if (argc != 5) { + throw std::runtime_error( + "usage: xbeam-native-station-file-test JOB.input " + "STREAM BAND OUTPUT_PREFIX"); + } + xbeam::RadioJob native(argv[1]); + J job = J::parse(native.normalized_json()); + int stream_id = std::stoi(argv[2]), band_id = std::stoi(argv[3]); + const auto& stream = id(job["datastreams"], stream_id); + const auto& band = id(stream["recorded_bands"], band_id); + const auto& frequency = id(job["frequencies"], band["frequency_id"]); + std::string source = stream.at("source_files").at(0); + std::ifstream in(source, std::ios::binary); + if (!in) { + throw std::runtime_error("cannot read recording"); + } + std::vector frame( + stream.at("frame_size_bytes").get()); + xbeam::VdifHeader first{}; + std::map> payloads; + std::map validity; + double rate = stream.at("sample_rate_hz"); + std::int64_t maximum = 0; + for (int n = 0; n < 4096; ++n) { + if (!in.read(reinterpret_cast(frame.data()), frame.size())) { + throw std::runtime_error("short bounded VDIF input"); + } + auto h = xbeam::parse_vdif_header(frame.data(), frame.size()); + if (n == 0) { + first = h; + } + if (h.frame_length_bytes != frame.size() + || h.thread_id != first.thread_id + || h.samples_per_frame != first.samples_per_frame + || h.reference_epoch != first.reference_epoch) { + throw std::runtime_error( + "bounded frontend test requires consistent frame/thread geometry"); + } + const auto fps = std::llround(rate / first.samples_per_frame); + if (double(fps) * first.samples_per_frame != rate) { + throw std::runtime_error( + "nonintegral frames-per-second in bounded fixture"); + } + std::int64_t slot = + (std::int64_t(h.seconds_from_epoch) - first.seconds_from_epoch) + * fps + + std::int64_t(h.frame_number) - first.frame_number; + if (slot < 0 || slot > 16384) { + throw std::runtime_error( + "bounded fixture timeline exceeds limit"); + } + if (!payloads.count(slot) || (!validity[slot] && !h.invalid)) { + payloads[slot] = + std::vector(frame.begin() + 32, frame.end()); + validity[slot] = !h.invalid; + } + maximum = std::max(maximum, slot); + } + const int samples_per_frame = first.samples_per_frame, + payload_bytes = first.frame_length_bytes - 32; + std::vector payload((maximum + 1) * payload_bytes, 0), + valid(maximum + 1, 0); + for (const auto& row : payloads) { + std::copy( + row.second.begin(), + row.second.end(), + payload.begin() + row.first * payload_bytes); + valid[row.first] = validity[row.first]; + } + std::tm epoch{}; + epoch.tm_year = 100 + first.reference_epoch / 2; + epoch.tm_mon = (first.reference_epoch % 2) * 6; + epoch.tm_mday = 1; + const auto unix_epoch = ::timegm(&epoch); + const auto job_day = + (job.at("start_time").at("mjd").get() - 40587) + * 86400; + double source_seconds = + double( + std::int64_t(unix_epoch) + first.seconds_from_epoch - job_day) + - std::stod( + job.at("start_time").at("seconds_of_day").get()); + auto start_sample = std::llround(source_seconds * rate) + + std::int64_t(first.frame_number) * samples_per_frame; + const int rows = 8, + fft = frequency.at("channel_count").get() + * (stream.at("sampling") == "real" ? 2 : 1); + std::vector starts(rows); + double origin = std::max( + 0., + std::stod( + job.at("scans").at(0).at("start_offset_s").get())); + // Select a useful bounded test interval from the source extent and model, + // rather than accepting spectra from an all-invalid leading window. + const double source_midpoint = + (double(start_sample) + + double(maximum + 1) * samples_per_frame / 2.) + / rate; + double trial = std::max(origin, source_midpoint); + for (int iteration = 0; iteration < 4; ++iteration) { + auto pivot = native.source_blocks( + job.at("scans").at(0).at("identifier"), + stream_id, + band_id, + &trial, + 1); + trial = source_midpoint + pivot[0].centre_delay; + } + for (int i = 0; i < rows; ++i) { + starts[i] = trial + double(i) * fft / rate; + } + auto blocks = native.source_blocks( + job.at("scans").at(0).at("identifier"), + stream_id, + band_id, + starts.data(), + rows); + std::vector parameters(rows); + for (int i = 0; i < rows; ++i) { + auto& p = parameters[i]; + auto& b = blocks[i]; + p.output_index = b.output_sample; + p.coarse_sample = b.coarse_sample; + p.block_start = b.block_start; + p.delay_zero = b.delay_zero; + p.delay_step = b.delay_step; + p.delay_quadratic = b.delay_quadratic; + p.residual = b.residual; + p.valid_stop = fft; + } + xbeam::StationLayout layout{}; + layout.rows = rows; + layout.fft_samples = fft; + layout.channels = frequency.at("channel_count"); + layout.complex_input = stream.at("sampling") != "real"; + layout.mapping = frequency.at("sideband") == "USB" ? 0 + : layout.complex_input ? 2 + : 1; + if (stream.at("sampling") == "complex_dsb") { + layout.mapping = 3; + } + layout.validity_stride = samples_per_frame; + layout.samples_per_frame = samples_per_frame; + layout.payload_bytes = payload_bytes; + layout.channel_count = 1 << first.log2_channels; + layout.channel_index = band_id % layout.channel_count; + layout.source_samples = (maximum + 1) * samples_per_frame; + layout.source_start = start_sample; + layout.sample_rate = rate; + double sign = stream.at("sampling") == "complex" + && frequency.at("sideband") == "LSB" + ? -1. + : 1.; + layout.reference_hz = sign + * (frequency.at("band_edge_hz").get() + + (stream.at("sampling") == "complex_dsb" + ? .5 * frequency.at("bandwidth_hz").get() + : 0.)); + layout.lo_hz = sign * band.at("lo_offset_hz").get(); + Buffer ds(payload.size()), dv(valid.size()), + dp(parameters.size() * sizeof(parameters[0])); + check(cudaMemcpy( + ds.p, payload.data(), payload.size(), cudaMemcpyHostToDevice)); + check(cudaMemcpy( + dv.p, valid.data(), valid.size(), cudaMemcpyHostToDevice)); + check(cudaMemcpy( + dp.p, + parameters.data(), + parameters.size() * sizeof(parameters[0]), + cudaMemcpyHostToDevice)); + xbeam::CudaStation station(layout); + station.submit( + reinterpret_cast(ds.p), + reinterpret_cast(dv.p), + reinterpret_cast(dp.p)); + std::vector spectra(std::size_t(rows) * layout.channels * 2); + std::vector weights(rows); + station.download(spectra.data(), weights.data()); + std::string prefix = argv[4]; + std::ofstream out(prefix + ".spectra", std::ios::binary); + out.write(reinterpret_cast(spectra.data()), spectra.size() * 4); + out.close(); + if (!out) { + throw std::runtime_error("spectrum write failed"); + } + J metadata = {{"source", source}, + {"stream", stream_id}, + {"band", band_id}, + {"rows", rows}, + {"channels", layout.channels}, + {"starts", starts}, + {"weights", weights}, + {"source_start_sample", start_sample}, + {"source_samples", layout.source_samples}, + {"channel_index", layout.channel_index}, + {"frames", maximum + 1}}; + std::ofstream meta(prefix + ".json"); + meta << metadata.dump(2); + meta.close(); + if (!meta) { + throw std::runtime_error("metadata write failed"); + } + std::ofstream bytes(prefix + ".payload", std::ios::binary); + bytes.write(reinterpret_cast(payload.data()), payload.size()); + bytes.close(); + std::ofstream validity_out(prefix + ".validity", std::ios::binary); + validity_out.write(reinterpret_cast(valid.data()), valid.size()); + validity_out.close(); + std::cout << metadata.dump() << '\n'; + return 0; + } catch (const std::exception& e) { + std::cerr << "ERROR: " << e.what() << '\n'; + return 1; + } +} diff --git a/src/cuphoton/xbeam/native/cuda/station_test.cpp b/src/cuphoton/xbeam/native/cuda/station_test.cpp new file mode 100644 index 00000000..163450aa --- /dev/null +++ b/src/cuphoton/xbeam/native/cuda/station_test.cpp @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/cuda.hpp" +#include +#include +#include +#include +#include +#include +namespace { +void check(cudaError_t code) { + if (code != cudaSuccess) { + throw std::runtime_error(cudaGetErrorString(code)); + } +} +struct Buffer { + void* p = nullptr; + explicit Buffer(std::size_t n) { + check(cudaMalloc(&p, n)); + } + ~Buffer() { + if (p) { + cudaFree(p); + } + } +}; +} // namespace +int main() { + try { + for (int n : {8000, 262144}) { + for (int pattern : {0x55, 0xe4}) { + const int rows = 8, frame = n == 8000 ? 8000 : 32768; + std::vector source( + std::size_t(rows) * n / 4, pattern), + valid(std::size_t(rows) * n / frame, 1); + std::vector parameters(rows); + for (int row = 0; row < rows; ++row) { + parameters[row].output_index = std::int64_t(row) * n; + parameters[row].valid_stop = n; + } + Buffer device_source(source.size()), device_valid(valid.size()), + device_parameters( + parameters.size() * sizeof(parameters[0])); + check(cudaMemcpy( + device_source.p, + source.data(), + source.size(), + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + device_valid.p, + valid.data(), + valid.size(), + cudaMemcpyHostToDevice)); + check(cudaMemcpy( + device_parameters.p, + parameters.data(), + parameters.size() * sizeof(parameters[0]), + cudaMemcpyHostToDevice)); + xbeam::StationLayout layout{}; + layout.rows = rows; + layout.fft_samples = n; + layout.channels = n / 2; + layout.mapping = 0; + layout.validity_stride = frame; + layout.samples_per_frame = frame; + layout.payload_bytes = frame / 4; + layout.channel_count = 1; + layout.channel_index = 0; + layout.source_samples = std::int64_t(rows) * n; + layout.sample_rate = n == 8000 ? 125000000. : 4096000000.; + xbeam::CudaStation station(layout); + station.submit( + reinterpret_cast(device_source.p), + reinterpret_cast(device_valid.p), + reinterpret_cast(device_parameters.p)); + std::vector output( + std::size_t(rows) * layout.channels * 2); + std::vector weights(rows); + station.download(output.data(), weights.data()); + const float levels[] = {-3.3359f, -1.f, 1.f, 3.3359f}; + double maximum = 0; + for (int row = 0; row < rows; ++row) { + if (weights[row] != 1.) { + throw std::runtime_error( + "native frontend weight mismatch"); + } + for (int channel = 0; channel < layout.channels; + ++channel) { + std::complex expected = 0.; + if (channel % (n / 4) == 0) { + for (int sample = 0; sample < 4; ++sample) { + double phase = -2. * std::acos(-1.) + * (channel / (n / 4)) * sample / 4.; + expected += + double( + levels[(pattern >> (sample * 2)) & 3]) + * std::complex( + std::cos(phase), std::sin(phase)); + } + expected *= double(n) / 4.; + } + auto index = + (std::size_t(row) * layout.channels + channel) * 2; + double error = std::abs( + std::complex( + output[index], output[index + 1]) + - expected); + if (!std::isfinite(error)) { + throw std::runtime_error( + "nonfinite native frontend output"); + } + maximum = std::max(maximum, error); + if (error > 1e-3 + 2e-6 * std::abs(expected)) { + throw std::runtime_error( + "native frontend spectrum mismatch"); + } + } + } + std::cout << "fft=" << n << " pattern=" << pattern + << " maximum_error=" << maximum << " PASS\n"; + } + } + return 0; + } catch (const std::exception& e) { + std::cerr << "ERROR: " << e.what() << '\n'; + return 1; + } +} diff --git a/src/cuphoton/xbeam/native/include/xbeam/cuda.hpp b/src/cuphoton/xbeam/native/include/xbeam/cuda.hpp new file mode 100644 index 00000000..1180ddf3 --- /dev/null +++ b/src/cuphoton/xbeam/native/include/xbeam/cuda.hpp @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include +#include +namespace xbeam { +struct StationLayout { + int rows, fft_samples, channels, mapping; + int validity_stride, samples_per_frame, payload_bytes, channel_count, + channel_index; + bool complex_input = false, authoritative_validity = false; + std::int64_t source_samples = 0, source_start = 0; + double sample_rate = 0, reference_hz = 0, lo_hz = 0, wall_lo_cycles = 0; +}; +// Binary-compatible with the existing frontend's per-FFT metadata. +struct BlockParameters { + std::int64_t output_index, coarse_sample; + double block_start, delay_zero, delay_step, delay_quadratic, residual; + std::int64_t source_offset, source_cutover, valid_start, valid_stop; +}; +static_assert(sizeof(BlockParameters) == 88, "block parameter ABI changed"); +class CudaStation { +public: + explicit CudaStation(const StationLayout&); + ~CudaStation(); + CudaStation(const CudaStation&) = delete; + CudaStation& operator=(const CudaStation&) = delete; + // Arguments are device addresses; caller retains them until stream completion. + void submit( + std::uintptr_t source, + std::uintptr_t validity, + std::uintptr_t parameters); + void download(float* interleaved, double* weights); + void source_window(std::int64_t first, std::int64_t count); + std::uintptr_t stream() const; + std::uintptr_t spectra() const; + std::uintptr_t weights() const; + +private: + struct Impl; + std::unique_ptr impl; +}; +} // namespace xbeam +namespace xbeam { +struct ProductLayout { + int channels, averaging, groups; + std::int64_t stride_a, stride_b; + double inverse_normalization; + bool multiply_weights; +}; +void submit_product( + const ProductLayout&, + std::uintptr_t spectrum_a, + std::uintptr_t spectrum_b, + std::uintptr_t weight_a, + std::uintptr_t weight_b, + std::uintptr_t group_starts, + std::uintptr_t group_lengths, + std::uintptr_t output_offsets, + std::uintptr_t weight_offsets, + std::uintptr_t output, + std::uintptr_t weight_sum, + std::uintptr_t stream); +} // namespace xbeam + +#include +#include +namespace xbeam { +struct NativeReadChunk { + std::uint64_t generation; + std::size_t index, offset, size; +}; +// How a recording is read. Buffered and Direct bounce through pinned host memory +// (Direct opens with O_DIRECT, bypassing the page cache); GpuDirect uses cuFile. +enum class ReadMode { Buffered, Direct, GpuDirect }; +class CudaRecordingInput { +public: + CudaRecordingInput( + const std::string& source, + std::int64_t frame_bytes, + std::int64_t payload_bytes, + ReadMode mode, + std::size_t chunk_bytes = 16 * 1024 * 1024, + bool shared_staging = false); + ~CudaRecordingInput(); + std::vector begin( + std::int64_t first, + std::int64_t stop, + const std::vector& validity); + void read_chunk(const NativeReadChunk&); + void finish(); + void read( + std::int64_t first, + std::int64_t stop, + const std::vector& validity); + std::uintptr_t payload() const; + std::uintptr_t validity() const; + +private: + struct Impl; + std::unique_ptr impl; +}; +void unpack_frames( + const void* raw, + void* payload, + std::int64_t frames, + std::int64_t frame_bytes, + std::int64_t payload_bytes, + std::int64_t prefix, + std::uintptr_t stream); +void unpack_frame_chunk( + const void* raw, + void* payload, + std::int64_t first_frame, + std::int64_t frames, + std::int64_t frame_bytes, + std::int64_t payload_bytes, + std::int64_t file_offset, + std::int64_t read_bytes, + std::uintptr_t stream); +} // namespace xbeam + +namespace xbeam { +struct VisibilityProduct { + std::uintptr_t spectrum_a, spectrum_b, weight_a, weight_b; + std::int64_t stride_a, stride_b; + int channels, averaging; + double inverse_normalization; + bool multiply_weights; +}; +class CudaProducts { +public: + explicit CudaProducts(int products_per_pack = 4, bool reuse_sources = true); + ~CudaProducts(); + void configure(const std::vector&); + void submit( + int groups, + std::uintptr_t group_starts, + std::uintptr_t group_lengths, + std::uintptr_t output_offsets, + std::uintptr_t weight_offsets, + std::uintptr_t output, + std::uintptr_t weights, + std::uintptr_t stream); + +private: + struct Impl; + std::unique_ptr impl; +}; +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/include/xbeam/execution.hpp b/src/cuphoton/xbeam/native/include/xbeam/execution.hpp new file mode 100644 index 00000000..4bd7e40e --- /dev/null +++ b/src/cuphoton/xbeam/native/include/xbeam/execution.hpp @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include +#include +namespace xbeam { +// The execution library owns all planning, IO, CUDA work, and output. +std::string correlate_radio(const std::vector& arguments); +// Optional Cython build bridge; CPU-only builds fail explicitly. +std::string correlate_native(const std::vector& arguments); +bool native_cuda_available(); +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/include/xbeam/radio.hpp b/src/cuphoton/xbeam/native/include/xbeam/radio.hpp new file mode 100644 index 00000000..39962ba3 --- /dev/null +++ b/src/cuphoton/xbeam/native/include/xbeam/radio.hpp @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include +#include +#include +#include +#include +namespace xbeam { +class RadioError : public std::runtime_error { + using std::runtime_error::runtime_error; +}; +struct VdifHeader { + bool invalid, complex_data; + std::uint32_t seconds_from_epoch, reference_epoch, frame_number; + std::uint32_t version, log2_channels, frame_length_bytes; + std::uint32_t bits_per_sample, thread_id, station_id, extended_data_version; + std::array extended_words; + std::uint64_t samples_per_frame; +}; +VdifHeader parse_vdif_header( + const unsigned char*, + std::size_t, + bool strict = true, + bool require_two_bit = true); +struct RecordingSummary { + std::uint64_t frames = 0, invalid_frames = 0, source_bytes = 0; + std::uint32_t frame_bytes = 0; + std::array thread_frames{}; +}; +RecordingSummary inspect_vdif(const std::string&, bool require_two_bit = true); +std::string inspect_vdif_json(const std::string&); +} // namespace xbeam + +namespace xbeam { +struct SwinMetadata { + std::int32_t baseline, configuration_id, source_id, frequency_id; + std::uint32_t mjd, pulsar_bin; + double seconds, weight, u, v, w; + char polarization_a, polarization_b; +}; +std::string encode_swin( + const SwinMetadata&, const float* interleaved, std::size_t channels); +void write_swin_file(const std::string& path, const std::string& records); +} // namespace xbeam + +namespace xbeam { +struct RequestGeometry { + std::int64_t ratio_numerator, ratio_denominator, fft_samples; + std::int64_t ffts_per_request, alignment_bytes, word_bytes; +}; +struct RequestCursor { + bool valid = false; + std::int64_t origin_bytes = 0, skipped_ffts = 0, decoder_delta_samples = 0; +}; +RequestCursor request_cursor( + std::int64_t local_samples, + double drift_seconds, + const RequestGeometry&, + double sample_rate); +} // namespace xbeam + +#include +namespace xbeam { +struct TextEntry { + std::string key, value; + std::size_t line_number; +}; +std::vector parse_job_text(const std::string& text); +} // namespace xbeam +namespace xbeam { +std::string load_interop_job_json(const std::string& path); +} +#include +#include +namespace xbeam { +struct IndexFrame { + std::uint64_t timestamp, physical_frame; + std::uint16_t thread; + std::uint8_t state; +}; +struct PhysicalIndexRun { + std::uint64_t first, stop, packed_timestamp; +}; +class IndexReader { +public: + IndexReader(const std::string& source, const std::string& index); + ~IndexReader(); + IndexReader(const IndexReader&) = delete; + IndexReader& operator=(const IndexReader&) = delete; + IndexReader(IndexReader&&) noexcept; + std::string metadata_json() const; + std::vector physical_runs() const; + std::vector frame_validity( + std::uint64_t first, std::uint64_t count) const; + std::vector query( + std::uint16_t thread, std::uint64_t first, std::uint64_t stop) const; + +private: + struct Impl; + std::unique_ptr impl; +}; +} // namespace xbeam +namespace xbeam { +struct SourceBlock { + std::int64_t fft_samples, output_sample, coarse_sample; + double centre_delay; + double block_start = 0, delay_zero = 0, delay_step = 0, delay_quadratic = 0, + residual = 0; +}; +struct JobRequestInputs; +class RadioJob { +public: + explicit RadioJob(const std::string& input); + ~RadioJob(); + RadioJob(const RadioJob&) = delete; + RadioJob& operator=(const RadioJob&) = delete; + std::string normalized_json() const; + std::int64_t frame_slot( + std::int64_t frames_per_second, int reference_epoch) const; + std::pair subintegration_ratio() const; + SwinMetadata record_metadata( + const std::string& scan, + std::int64_t integration, + int station_a, + int station_b, + int frequency, + char pol_a, + char pol_b) const; + std::vector source_blocks( + const std::string& scan, + int stream, + int band, + const double* starts, + std::size_t count, + bool literal_endpoint = false, + bool phase_parameters = true) const; + JobRequestInputs requests( + const std::string& scan, + int stream, + int band, + const RequestGeometry&, + std::int64_t first, + std::int64_t stop, + std::int64_t timeline_count, + std::int64_t job_start_slot, + std::int64_t timestamp_origin, + bool endpoint_mean, + std::int64_t maximum_requests) const; + +private: + struct Impl; + std::unique_ptr impl; +}; +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/include/xbeam/recording.hpp b/src/cuphoton/xbeam/native/include/xbeam/recording.hpp new file mode 100644 index 00000000..b19bbf0c --- /dev/null +++ b/src/cuphoton/xbeam/native/include/xbeam/recording.hpp @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "xbeam/window.hpp" +#include +namespace xbeam { +struct RecordingGeometry { + std::int64_t sample_rate, frames_per_second, samples_per_frame, frame_count; + std::int64_t frame_bytes, payload_bytes, job_slot; + int channel_count, thread_id; + WindowGeometry windows; + RequestGeometry request; + NormalExtent extent; +}; +struct RadioReplayBatch { + std::vector blocks; + std::vector replay; + int channel_index; +}; +class RadioRecording { +public: + RadioRecording(const RadioJob&, int stream, const std::string& index); + // Adopts an already validated sidecar instead of validating it again. + RadioRecording(const RadioJob&, int stream, IndexReader&& index); + ~RadioRecording(); + const RecordingGeometry& geometry() const; + std::vector frame_validity( + std::int64_t first, std::int64_t count) const; + RadioReplayBatch plan( + const std::string& scan, + int band, + std::int64_t first, + std::int64_t stop, + std::int64_t timeline_count) const; + +private: + struct Impl; + std::unique_ptr impl; +}; +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/include/xbeam/replay.hpp b/src/cuphoton/xbeam/native/include/xbeam/replay.hpp new file mode 100644 index 00000000..9e465d6f --- /dev/null +++ b/src/cuphoton/xbeam/native/include/xbeam/replay.hpp @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "xbeam/radio.hpp" +#include "xbeam/timeline.hpp" +namespace xbeam { +struct RawRequestState { + std::vector anchors; + std::vector events; + std::int64_t allocation_frames, samples_per_frame, timestamp_origin; +}; +struct RequestFact { + std::int64_t first_fft, desired_sample; + std::int64_t active_count = 0; + double endpoint_drift; + std::vector ordinary_samples; +}; +struct JobRequestInputs { + std::int64_t first_fft, stop_fft, timeline_fft_count, timestamp_origin; + double sample_rate; + std::vector requests; + std::vector> output_samples; + std::vector> centre_delays, read_average; + std::vector left_delays; +}; +struct DiscardBatch { + std::vector allocations; + std::vector cursors; + std::vector valid, events; + std::vector> eligible; + std::vector excluded; +}; +RawRequestState raw_request_state( + const PhysicalTimeline&, + std::int64_t samples_per_frame, + std::int64_t timestamp_origin, + bool clock_frame_owner); +DiscardBatch compile_discards( + const RawRequestState&, + const std::vector&, + const RequestGeometry&, + double sample_rate, + std::int64_t frame_bytes, + std::int64_t control_word_bits); +struct WindowBounds { + std::int64_t nearest, valid_first, valid_stop; + bool before, after; +}; +WindowBounds window_bounds( + std::int64_t output, + std::int64_t head, + double delay_seconds, + std::int64_t sample_rate, + std::int64_t framed_extent, + std::int64_t decoded_extent, + std::int64_t fft_samples); +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/include/xbeam/series.hpp b/src/cuphoton/xbeam/native/include/xbeam/series.hpp new file mode 100644 index 00000000..f68b6ca5 --- /dev/null +++ b/src/cuphoton/xbeam/native/include/xbeam/series.hpp @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include +#include +#include +#include +namespace xbeam { +// HDF5 metadata is validated by the adapter; native code owns raw delivery and DSP. +class NativeSeries { +public: + explicit NativeSeries(const std::string& configuration); + ~NativeSeries(); + NativeSeries(const NativeSeries&) = delete; + NativeSeries& operator=(const NativeSeries&) = delete; + void run( + std::int64_t first, + int windows, + void* spectrum, + std::size_t spectrum_bytes, + void* lag, + std::size_t lag_bytes, + std::int64_t* statistics, + std::size_t statistics_count, + std::int64_t* coverage, + std::size_t coverage_count); + +private: + struct Impl; + std::unique_ptr impl; +}; +// Optional-build bridge: CPU-only extensions fail before opening the input. +NativeSeries* create_native_series(const std::string&); +void destroy_native_series(NativeSeries*); +void run_native_series( + NativeSeries*, + std::int64_t first, + int windows, + void* spectrum, + std::size_t spectrum_bytes, + void* lag, + std::size_t lag_bytes, + std::int64_t* statistics, + std::size_t statistics_count, + std::int64_t* coverage, + std::size_t coverage_count); +// Shared cuFile driver owner, also used by recording delivery. +void ensure_cufile_driver(); +// Uses the same generated XMAC kernels as radio execution. +void sum_series_spectra( + bool wide, + int rows, + int channels, + const void* a, + const void* b, + const double* weights, + double scale, + void* output, + double* exposure, + std::uintptr_t stream); +void pack_series_input( + const void* raw, + const unsigned char* mask, + int dtype, + bool wide, + int rows, + int samples, + int fft_samples, + std::int64_t hop, + void* output, + unsigned char* validity, + void* mask_spectrum, + int* error, + std::uintptr_t stream); +void series_statistics( + const unsigned char* a, + const unsigned char* b, + int rows, + int samples, + std::int64_t* statistics, + std::uintptr_t stream); +void add_series_spectrum( + bool wide, + int channels, + const void* input, + void* output, + std::uintptr_t stream); +void gather_series_lag( + bool wide, + int pairs, + int fft_samples, + int lag_first, + int lag_count, + const void* inverse, + void* output, + const void* mask_inverse, + std::int64_t* coverage, + std::uintptr_t stream); +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/include/xbeam/timeline.hpp b/src/cuphoton/xbeam/native/include/xbeam/timeline.hpp new file mode 100644 index 00000000..72455fb1 --- /dev/null +++ b/src/cuphoton/xbeam/native/include/xbeam/timeline.hpp @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include +#include +namespace xbeam { +struct PhysicalRun { + std::int64_t first, stop, timestamp; +}; +struct SampleSpan { + std::int64_t request_first, request_stop, physical_first; +}; +struct AnchorGap { + std::int64_t allocation, physical_boundary, first, stop; +}; +struct FrameInterval { + std::int64_t first, stop; +}; +class PhysicalTimeline { +public: + PhysicalTimeline( + const std::vector&, std::int64_t allocation_frames); + const std::vector& allocation_starts() const { + return allocation_starts_; + } + const std::vector& anchors() const { + return anchors_; + } + std::vector uncovered_heads() const; + std::vector deferred_gaps() const; + std::vector source_spans( + std::int64_t first, + std::int64_t count, + std::int64_t samples_per_frame) const; + std::vector latched_spans( + std::int64_t allocation, + std::int64_t first, + std::int64_t count, + std::int64_t samples_per_frame) const; + + const std::vector& physical_runs() const { + return runs; + } + std::int64_t allocation_frames() const { + return capacity; + } + +private: + std::vector runs; + std::vector physical_stops, timestamp_stops, + allocation_starts_, anchors_; + std::int64_t capacity; +}; +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/include/xbeam/window.hpp b/src/cuphoton/xbeam/native/include/xbeam/window.hpp new file mode 100644 index 00000000..4fdcf2ec --- /dev/null +++ b/src/cuphoton/xbeam/native/include/xbeam/window.hpp @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "xbeam/replay.hpp" +namespace xbeam { +struct WindowGeometry { + std::int64_t frames_per_fft, ffts_per_request, buffered_ffts, ring_segments; + std::int64_t allocation_frames, request_frames, read_buffer_ratio; +}; +// The subintegration is supplied as its exact decimal-derived rational. +WindowGeometry size_windows( + std::int64_t samples_per_frame, + std::int64_t fft_samples, + std::int64_t sample_rate, + std::int64_t subint_num, + std::int64_t subint_den, + std::int64_t buffer_factor, + std::int64_t data_segments, + std::int64_t buffered_ffts, + double reference_rate_allowance); +struct NormalExtent { + std::int64_t frame_bytes, payload_bytes, request_frames, read_buffer_ratio; + std::int64_t ratio_num, ratio_den, samples_per_frame; + std::int64_t framed_admission_samples, decoded_samples, allocation_bytes; +}; +NormalExtent normal_extent( + std::int64_t frame_bytes, + std::int64_t payload_bytes, + std::int64_t request_frames, + std::int64_t read_buffer_ratio, + std::int64_t ratio_num, + std::int64_t ratio_den); +std::int64_t leading_head( + std::int64_t output, + double endpoint_delay, + std::int64_t sample_rate, + std::int64_t samples_per_frame, + std::int64_t timestamp_quantum_ns); +struct ReturnedWindow { + std::int64_t fft, output, logical_head, physical_head; + double centre_delay; + std::int64_t framed_extent, decoded_extent; + bool published, clamp_negative; +}; +// A bounded normal-return contract. Initial returns and EOF are explicit inputs. +std::vector returned_windows( + const PhysicalTimeline&, + const RawRequestState&, + const JobRequestInputs&, + const RequestGeometry&, + const NormalExtent&, + std::int64_t control_bits, + std::int64_t timestamp_quantum_ns, + std::int64_t known_source_stop); +} // namespace xbeam +namespace xbeam { +struct ReplayRow { + std::int64_t fft, physical_start, timing_displacement, valid_first, + valid_stop; + bool accepted; +}; +// Produces source coordinates and decoder masks; frame payload validity is +// still supplied independently to the GPU frontend. +std::vector compile_replay_rows( + const std::vector&, + std::int64_t fft_samples, + std::int64_t sample_rate, + std::int64_t known_source_stop); +} // namespace xbeam + +namespace xbeam { +std::int64_t packed_frame_slot(std::uint64_t timestamp, std::int64_t fps); +std::int64_t reference_epoch_seconds(int epoch); +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/include/xbeam/work_queue.hpp b/src/cuphoton/xbeam/native/include/xbeam/work_queue.hpp new file mode 100644 index 00000000..f3da9f6e --- /dev/null +++ b/src/cuphoton/xbeam/native/include/xbeam/work_queue.hpp @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +namespace xbeam { +class WorkQueue { + std::mutex mutex; + std::condition_variable ready; + std::queue> pending; + std::vector workers; + bool stopping = false; + +public: + explicit WorkQueue(unsigned count) { + if (count == 0) { + throw std::invalid_argument("work queue needs at least one worker"); + } + try { + for (unsigned k = 0; k < count; ++k) { + workers.emplace_back([this] { + for (;;) { + std::function task; + { + std::unique_lock lock(mutex); + ready.wait(lock, [this] { + return stopping || !pending.empty(); + }); + if (stopping && pending.empty()) { + return; + } + task = std::move(pending.front()); + pending.pop(); + } + task(); + } + }); + } + } catch (...) { + { + std::lock_guard lock(mutex); + stopping = true; + } + ready.notify_all(); + for (auto& worker : workers) { + worker.join(); + } + throw; + } + } + ~WorkQueue() { + { + std::lock_guard lock(mutex); + stopping = true; + } + ready.notify_all(); + for (auto& worker : workers) { + worker.join(); + } + } + std::future submit(std::function function) { + auto task = + std::make_shared>(std::move(function)); + auto future = task->get_future(); + { + std::lock_guard lock(mutex); + pending.emplace([task] { (*task)(); }); + } + ready.notify_one(); + return future; + } +}; +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/series.py b/src/cuphoton/xbeam/native/series.py new file mode 100644 index 00000000..2dc284b5 --- /dev/null +++ b/src/cuphoton/xbeam/native/series.py @@ -0,0 +1,173 @@ +# SPDX-License-Identifier: Apache-2.0 +"""HDF5 metadata adapter for native time-series execution. + +Only metadata and owned host results cross Python. File reads, masks, FFTs, +products, and lag reconstruction execute in the native CUDA library. +""" + +import math +import os +from importlib import import_module + +import numpy as np + +from ..adapters.hdf5 import Hdf5Source +from ..adapters.hdf5_layout import extent +from ..core.series import SeriesResult, plan_series_shards +from ..runtime.uploads import bounded_upload_workers + + +def _configuration(source, spec): + if not isinstance(source, Hdf5Source): + raise TypeError("native series execution requires an Hdf5Source") + if not math.isfinite(source.sample_rate_hz) or source.sample_rate_hz <= 0: + raise ValueError("sample_rate_hz must be finite and positive") + if not math.isfinite(source.origin_seconds): + raise ValueError("origin_seconds must be finite") + if source.io_queue_depth != 2: + raise ValueError("native series currently requires io_queue_depth=2") + file = source._file + if not file or file.driver != "sec2": + raise ValueError("native series requires an open HDF5 sec2 file") + signals = tuple(dict.fromkeys(s for pair in spec.pairs for s in pair)) + if any(s not in source.signal_ids for s in signals): + raise ValueError("requested pair contains an unavailable signal") + kinds = { + np.dtype("float32"): 0, + np.dtype("float64"): 1, + np.dtype("complex64"): 2, + np.dtype("complex128"): 3, + } + descriptors = [] + dtypes = [] + for signal in signals: + dataset = source._datasets[signal] + info = extent(dataset, file_id=file.id) + if info is None or info.dtype not in kinds: + raise ValueError( + "native series requires allocated, native contiguous " + "float32/float64/complex64/complex128 HDF5 arrays" + ) + mask = source._validity.get(signal) + mask_info = None if mask is None else extent(mask, file_id=file.id) + if mask is not None and mask_info is None: + raise ValueError( + "native validity must be allocated and contiguous" + ) + descriptors.append( + dict( + offset=info.offset, + dtype=kinds[info.dtype], + mask_offset=-1 if mask is None else mask_info.offset, + ) + ) + dtypes.append(info.dtype) + wide = spec.precision == "float64" or ( + spec.precision is None + and any(d.itemsize > (8 if d.kind == "c" else 4) for d in dtypes) + ) + stat = os.fstat(file.id.get_vfd_handle()) + workers = bounded_upload_workers( + source.upload_workers, automatic=2, available=64 + ) + return dict( + path=os.path.abspath(file.filename), + device=stat.st_dev, + inode=stat.st_ino, + io=source.io_backend, + gpu=-1, + window_samples=spec.window_samples, + fft_samples=spec.fft_samples, + windows_per_result=spec.windows_per_result, + batch_windows=spec.batch_windows, + sample_count=source.sample_count, + hop_samples=spec.hop_samples, + wide=wide, + want_lag=spec.output != "spectrum", + want_spectrum=spec.output != "lag", + linear=spec.lag_mode == "linear", + inputs=descriptors, + pairs=[(signals.index(a), signals.index(b)) for a, b in spec.pairs], + upload_workers=workers, + read_buffer_bytes=source.read_buffer_bytes, + prefetch=bool(spec.prefetch_batches), + ) + + +def correlate_source(source, spec): + """Stream host-owned results using the optional native CUDA executor.""" + config = _configuration(source, spec) + shard = plan_series_shards(source.sample_count, spec)[spec.shard_index] + if shard.first_result == shard.stop_result: + return + from . import cuda_available + + if not cuda_available(): + raise RuntimeError( + "native CUDA execution was not built; enable " + "CUPHOTON_XBEAM_BUILD_EXT=1 and CUPHOTON_XBEAM_NATIVE_CUDA=1" + ) + executor = import_module("cuphoton.xbeam.native._binding").SeriesExecutor( + config + ) + n, fft_n = spec.window_samples, spec.fft_samples + count, group_n = len(spec.pairs), spec.windows_per_result + dtype = np.complex128 if config["wide"] else np.complex64 + lags = ( + np.arange(-(n - 1), n, dtype=np.int64) + if spec.lag_mode == "linear" + else np.arange(-(n // 2), n - n // 2, dtype=np.int64) + ) + frequencies = np.fft.fftfreq(fft_n, 1 / source.sample_rate_hz) + try: + for ordinal in range(shard.first_result, shard.stop_result): + starts = ( + ordinal * group_n + np.arange(group_n, dtype=np.int64) + ) * spec.hop_samples + spectrum = ( + np.empty((count, fft_n), dtype) + if config["want_spectrum"] + else np.empty(0, dtype) + ) + lag = ( + np.empty((count, len(lags)), dtype) + if config["want_lag"] + else np.empty(0, dtype) + ) + coverage = ( + np.empty((count, len(lags)), np.int64) + if config["want_lag"] + else np.empty(0, np.int64) + ) + statistics = np.empty((count, 2), np.int64) + executor.run( + int(starts[0]), + group_n, + spectrum.reshape(-1).view(np.uint8), + lag.reshape(-1).view(np.uint8), + statistics.reshape(-1), + coverage.reshape(-1), + ) + yield SeriesResult( + result_index=ordinal, + pairs=spec.pairs, + sample_start=int(starts[0]), + sample_stop=int(starts[-1]) + n, + window_starts=starts, + sample_rate_hz=source.sample_rate_hz, + origin_seconds=source.origin_seconds, + contributing_windows=statistics[:, 0].copy(), + simultaneous_valid_samples=statistics[:, 1].copy(), + valid=statistics[:, 0] > 0, + spectrum=spectrum if config["want_spectrum"] else None, + frequencies_hz=frequencies.copy() + if config["want_spectrum"] + else None, + lag=lag if config["want_lag"] else None, + lag_samples=lags.copy() if config["want_lag"] else None, + lag_valid_samples=coverage if config["want_lag"] else None, + fft_samples=fft_n, + lag_mode=spec.lag_mode, + ) + finally: + executor.close() diff --git a/src/cuphoton/xbeam/native/src/execution_bridge.cpp b/src/cuphoton/xbeam/native/src/execution_bridge.cpp new file mode 100644 index 00000000..a2cbbc9a --- /dev/null +++ b/src/cuphoton/xbeam/native/src/execution_bridge.cpp @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/execution.hpp" +#include "xbeam/radio.hpp" +#include "xbeam/series.hpp" +namespace xbeam { +bool native_cuda_available() { +#ifdef XBEAM_BIND_CUDA + return true; +#else + return false; +#endif +} +std::string correlate_native(const std::vector& arguments) { +#ifdef XBEAM_BIND_CUDA + return correlate_radio(arguments); +#else + (void)arguments; + throw RadioError( + "native CUDA execution was not built; enable " + "CUPHOTON_XBEAM_NATIVE_CUDA=1 during the opt-in extension build"); +#endif +} +NativeSeries* create_native_series(const std::string& configuration) { +#ifdef XBEAM_BIND_CUDA + return new NativeSeries(configuration); +#else + (void)configuration; + throw RadioError( + "native CUDA execution was not built; enable CUPHOTON_XBEAM_NATIVE_CUDA=1"); +#endif +} +void destroy_native_series(NativeSeries* series) { +#ifdef XBEAM_BIND_CUDA + delete series; +#else + (void)series; +#endif +} +void run_native_series( + NativeSeries* series, + std::int64_t first, + int windows, + void* spectrum, + std::size_t spectrum_bytes, + void* lag, + std::size_t lag_bytes, + std::int64_t* statistics, + std::size_t statistics_count, + std::int64_t* coverage, + std::size_t coverage_count) { +#ifdef XBEAM_BIND_CUDA + series->run( + first, + windows, + spectrum, + spectrum_bytes, + lag, + lag_bytes, + statistics, + statistics_count, + coverage, + coverage_count); +#else + (void)series; + (void)first; + (void)windows; + (void)spectrum; + (void)spectrum_bytes; + (void)lag; + (void)lag_bytes; + (void)statistics; + (void)statistics_count; + (void)coverage; + (void)coverage_count; + throw RadioError("native CUDA execution was not built"); +#endif +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/src/index.cpp b/src/cuphoton/xbeam/native/src/index.cpp new file mode 100644 index 00000000..103e1155 --- /dev/null +++ b/src/cuphoton/xbeam/native/src/index.cpp @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/radio.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace xbeam { +namespace { +std::uint64_t u64(const unsigned char* p) { + std::uint64_t v = 0; + for (int i = 7; i >= 0; --i) { + v = (v << 8) | p[i]; + } + return v; +} +std::uint32_t u32(const unsigned char* p) { + return std::uint32_t(u64(p) & 0xffffffff); +} +std::uint64_t align(std::uint64_t n) { + if (n > UINT64_MAX - 4095) { + throw RadioError("index extent overflow"); + } + return (n + 4095) / 4096 * 4096; +} +struct File { + int fd = -1; + struct stat identity{}; + std::string path; + explicit File(const std::string& p) + : path(p) { + if (p.find('\0') != std::string::npos) { + throw RadioError("path contains NUL"); + } + fd = ::open(p.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) { + throw RadioError("cannot open index/source: " + p); + } + if (::fstat(fd, &identity) || !S_ISREG(identity.st_mode)) { + ::close(fd); + fd = -1; + throw RadioError("index/source is not a regular file"); + } + } + ~File() { + if (fd >= 0) { + ::close(fd); + } + } + void unchanged() const { + struct stat after{}, named{}; + if (::stat(path.c_str(), &named) || named.st_dev != identity.st_dev + || named.st_ino != identity.st_ino) { + throw RadioError("index/source path was replaced"); + } + if (::fstat(fd, &after) || after.st_size != identity.st_size + || after.st_mtim.tv_sec != identity.st_mtim.tv_sec + || after.st_mtim.tv_nsec != identity.st_mtim.tv_nsec + || after.st_ctim.tv_sec != identity.st_ctim.tv_sec + || after.st_ctim.tv_nsec != identity.st_ctim.tv_nsec) { + throw RadioError("index/source changed during use"); + } + } + void read( + unsigned char* dst, std::size_t count, std::uint64_t offset) const { + std::size_t n = 0; + while (n < count) { + auto got = ::pread(fd, dst + n, count - n, offset + n); + if (got < 0 && errno == EINTR) { + continue; + } + if (got <= 0) { + throw RadioError("truncated index/source read"); + } + n += got; + } + } +}; +// Own a bounded read window so concurrent file changes cannot invalidate +// memory being inspected. Each query owns its cache and remains independent. +struct ReadCache { + const File& file; + std::array data; + std::uint64_t first = 0; + std::size_t cached = 0; + explicit ReadCache(const File& source) + : file(source) {} + const unsigned char* read(std::uint64_t offset, std::size_t count) { + const auto size = std::uint64_t(file.identity.st_size); + if (count > data.size() || offset > size || count > size - offset) { + throw RadioError("index read exceeds sidecar extent"); + } + if (offset < first || offset - first > cached + || count > cached - (offset - first)) { + first = offset; + cached = std::min(data.size(), size - first); + file.read(data.data(), cached, first); + } + return data.data() + (offset - first); + } +}; +std::string fingerprint(const File& f, std::uint64_t frame) { + auto n = std::min(frame, 4096); + std::vector data(n * 2); + f.read(data.data(), n, 0); + f.read(data.data() + n, n, f.identity.st_size - frame); + unsigned char digest[EVP_MAX_MD_SIZE]; + unsigned int length = 0; + if (!EVP_Digest( + data.data(), data.size(), digest, &length, EVP_sha256(), nullptr)) { + throw RadioError("source fingerprint failed"); + } + std::ostringstream out; + for (unsigned int i = 0; i < length; ++i) { + out << std::hex << std::setw(2) << std::setfill('0') + << unsigned(digest[i]); + } + return out.str(); +} +} // namespace +struct IndexReader::Impl { + File source, index; + std::size_t size; + std::uint64_t count, frame_count; + int version, stride; + nlohmann::json metadata; + std::vector validity; + std::vector physical; + Impl(const std::string& s, const std::string& p) + : source(s), + index(p), + size(index.identity.st_size) { + if (size < 65536) { + throw RadioError("truncated sidecar header"); + } + unsigned char prefix[16]; + index.read(prefix, 16, 0); + if (std::memcmp(prefix, "XCVDIX1\n", 8)) { + throw RadioError("invalid sidecar magic"); + } + // u32 requires eight readable bytes; prefix has both words available. + version = u32(prefix + 8); + auto bytes = std::uint32_t(prefix[12]) + | (std::uint32_t(prefix[13]) << 8) + | (std::uint32_t(prefix[14]) << 16) + | (std::uint32_t(prefix[15]) << 24); + if (version < 2 || version > 4 || bytes == 0 || bytes > 65536 - 16) { + throw RadioError("invalid sidecar version/metadata length"); + } + std::string text(bytes, '\0'); + index.read(reinterpret_cast(text.data()), bytes, 16); + metadata = nlohmann::json::parse(text); + if (metadata.at("schema_version") != version) { + throw RadioError("sidecar version fields disagree"); + } + count = metadata.at("entry_count").get(); + frame_count = metadata.at("frame_count").get(); + auto frame = metadata.at("frame_size_bytes").get(); + if (frame <= 32 || count != frame_count || count > UINT64_MAX / frame + || count * frame != std::uint64_t(source.identity.st_size) + || metadata.at("source_size_bytes") != source.identity.st_size) { + throw RadioError("sidecar source geometry is stale"); + } + const auto& provenance = metadata.at("source_provenance"); + auto mtime = std::int64_t(source.identity.st_mtim.tv_sec) * 1000000000 + + source.identity.st_mtim.tv_nsec; + auto ctime = std::int64_t(source.identity.st_ctim.tv_sec) * 1000000000 + + source.identity.st_ctim.tv_nsec; + if (provenance.at("version") != 1 + || provenance.at("size_bytes") != source.identity.st_size + || provenance.at("mtime_ns") != mtime + || provenance.at("ctime_ns") != ctime) { + throw RadioError("sidecar source provenance is stale"); + } + stride = version == 2 ? 16 : 24; + if (count > (UINT64_MAX - 65536) / stride) { + throw RadioError("sidecar extent overflow"); + } + auto stop = 65536 + count * stride; + auto run_offset = version == 4 ? align(stop) : stop; + auto runs = + metadata.value("physical_timestamp_run_count", std::uint64_t(0)); + if (runs > (UINT64_MAX - run_offset) / 24 + || size != align(run_offset + runs * 24)) { + throw RadioError("invalid sidecar extent"); + } + if (version < 4 && runs) { + throw RadioError("legacy sidecar declares physical runs"); + } + if (metadata.value("rejected_header_frame_count", 0) != 0 + || metadata.value("rejected_format_frame_count", 0) != 0) { + throw RadioError("sidecar contains rejected malformed frames"); + } + if (metadata.at("sample_fingerprint_sha256") + != fingerprint(source, frame)) { + throw RadioError("sidecar source fingerprint is stale"); + } + if (!metadata.at("threads").is_array() + || metadata.at("threads").empty()) { + throw RadioError("missing sidecar thread inventory"); + } + for (auto& thread : metadata.at("threads")) { + if (thread.at("frame_size_bytes") != frame) { + throw RadioError("invalid thread geometry"); + } + } + { + ReadCache rows(index); + std::uint64_t prev_stop = 0; + if (version == 4) { + if (metadata.at("physical_timestamp_run_layout") + != "physical-start-u64,physical-stop-u64,packed-timestamp-start-" + "u64" + || metadata.at("physical_timestamp_run_offset_bytes") + != run_offset) { + throw RadioError("invalid physical run layout"); + } + for (std::uint64_t i = 0; i < runs; ++i) { + auto p = rows.read(run_offset + i * 24, 24); + auto first = u64(p), last = u64(p + 8), time = u64(p + 16); + if (first >= last || last > count || first < prev_stop + || time >= (std::uint64_t(1) << 60) + || (time & 0xffffff) + (last - first) + > (std::uint64_t(1) << 24)) { + throw RadioError("invalid physical timestamp run"); + } + physical.push_back({first, last, time}); + prev_stop = last; + } + } + // Validate the index once before binary searches can trust its order. + std::vector seen(count / 8 + (count % 8 != 0), 0); + std::uint64_t previous = 0; + validity.resize(count); + for (std::uint64_t i = 0; i < count; ++i) { + auto entry = rows.read(65536 + i * stride, stride); + auto timestamp = u64(entry), location = u64(entry + 8); + auto ordinal = location & ((std::uint64_t(1) << 53) - 1); + auto state = version == 2 ? (((location >> 53) & 1) ? 8 : 0) + : u64(entry + 16); + if (timestamp >= (std::uint64_t(1) << 60) + || (i && timestamp < previous) || ordinal >= count) { + throw RadioError("invalid sidecar entry ordering/ordinal"); + } + if (seen[ordinal / 8] & (1u << (ordinal % 8))) { + throw RadioError("duplicate physical sidecar ordinal"); + } + seen[ordinal / 8] |= 1u << (ordinal % 8); + if (version >= 3 + && ((state & ~std::uint64_t(23)) + || bool(state & 7) != bool((location >> 53) & 1) + || (state & 6))) { + throw RadioError("invalid/rejected sidecar frame state"); + } + validity[ordinal] = state == 0; + previous = timestamp; + } + source.unchanged(); + index.unchanged(); + } + } +}; +IndexReader::IndexReader(const std::string& source, const std::string& index) + : impl(std::make_unique(source, index)) {} +IndexReader::~IndexReader() = default; +IndexReader::IndexReader(IndexReader&&) noexcept = default; +std::string IndexReader::metadata_json() const { + impl->source.unchanged(); + impl->index.unchanged(); + return impl->metadata.dump(); +} +std::vector IndexReader::query( + std::uint16_t thread, std::uint64_t first, std::uint64_t stop) const { + if (thread > 1023 || first > stop) { + throw RadioError("invalid sidecar query"); + } + impl->source.unchanged(); + impl->index.unchanged(); + ReadCache rows(impl->index); + auto row = [&](std::uint64_t i) { + return rows.read(65536 + i * impl->stride, impl->stride); + }; + auto lower = [&](std::uint64_t value) { + std::uint64_t lo = 0, hi = impl->count; + while (lo < hi) { + auto mid = lo + (hi - lo) / 2; + if (u64(row(mid)) < value) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo; + }; + std::vector result; + for (auto i = lower(first), end = lower(stop); i < end; ++i) { + auto entry = row(i); + auto loc = u64(entry + 8); + if ((loc >> 54) != thread) { + continue; + } + auto physical = loc & ((std::uint64_t(1) << 53) - 1); + auto state = + impl->version == 2 ? ((loc >> 53) & 1 ? 8 : 0) : u64(entry + 16); + if (physical >= impl->frame_count + || (impl->version >= 3 + && ((state & ~std::uint64_t(23)) + || bool(state & 7) != bool((loc >> 53) & 1)))) { + throw RadioError("invalid sidecar frame entry"); + } + result.push_back( + {u64(entry), physical, thread, static_cast(state)}); + } + impl->source.unchanged(); + impl->index.unchanged(); + return result; +} +} // namespace xbeam + +namespace xbeam { +std::vector IndexReader::physical_runs() const { + impl->source.unchanged(); + impl->index.unchanged(); + if (impl->version != 4) { + throw RadioError("physical replay requires schema-v4 sidecar"); + } + return impl->physical; +} +std::vector IndexReader::frame_validity( + std::uint64_t first, std::uint64_t count) const { + impl->source.unchanged(); + impl->index.unchanged(); + if (first > impl->count || count > impl->count - first) { + throw RadioError("physical validity query exceeds file"); + } + if (impl->metadata.at("threads").size() != 1) { + throw RadioError("physical replay requires a single recorded thread"); + } + return { + impl->validity.begin() + first, impl->validity.begin() + first + count}; +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/src/job.cpp b/src/cuphoton/xbeam/native/src/job.cpp new file mode 100644 index 00000000..b365aa8b --- /dev/null +++ b/src/cuphoton/xbeam/native/src/job.cpp @@ -0,0 +1,1249 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/radio.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace xbeam { +namespace { +using J = nlohmann::json; +using I = __int128; +I mul(I a, I b) { + I out; + if (__builtin_mul_overflow(a, b, &out)) { + throw RadioError("decimal overflow"); + } + return out; +} +I sum(I a, I b) { + I out; + if (__builtin_add_overflow(a, b, &out)) { + throw RadioError("decimal overflow"); + } + return out; +} +I power(int n) { + if (n < 0 || n > 37) { + throw RadioError("decimal scale out of range"); + } + I v = 1; + while (n--) { + v = mul(v, 10); + } + return v; +} +struct D { + I value = 0; + int scale = 0; + explicit D(const std::string& text) { + std::size_t pos = 0; + bool negative = false, dot = false; + int digits = 0; + if (pos < text.size() && (text[pos] == '-' || text[pos] == '+')) { + negative = text[pos++] == '-'; + } + for (; pos < text.size() && text[pos] != 'e' && text[pos] != 'E'; + ++pos) { + if (text[pos] == '.' && !dot) { + dot = true; + continue; + } + if (text[pos] < '0' || text[pos] > '9') { + throw RadioError("invalid decimal: " + text); + } + value = sum(mul(value, 10), text[pos] - '0'); + ++digits; + if (dot) { + ++scale; + } + } + if (!digits) { + throw RadioError("empty decimal"); + } + if (pos < text.size()) { + auto e = std::stoi(text.substr(pos + 1)); + if (e > 37 || e < -37) { + throw RadioError("decimal exponent out of range"); + } + scale -= e; + } + if (scale < 0) { + value = mul(value, power(-scale)); + scale = 0; + } + if (scale > 37) { + throw RadioError("decimal precision out of range"); + } + if (negative) { + value = -value; + } + } + D(I v, int s) + : value(v), + scale(s) {} + D operator+(const D& b) const { + int s = std::max(scale, b.scale); + return D( + sum(mul(value, power(s - scale)), mul(b.value, power(s - b.scale))), + s); + } + D operator-(const D& b) const { + return *this + D(-b.value, b.scale); + } + D times(I x) const { + return D(mul(value, x), scale); + } + std::string str() const { + I x = value; + bool neg = x < 0; + if (neg) { + x = -x; + } + std::string s; + do { + s.push_back(char('0' + x % 10)); + x /= 10; + } while (x); + std::reverse(s.begin(), s.end()); + if (scale) { + if (s.size() <= std::size_t(scale)) { + s = std::string(scale + 1 - s.size(), '0') + s; + } + s.insert(s.size() - scale, "."); + } + return (neg ? "-" : "") + s; + } +}; +struct Doc { + std::vector entries; + std::vector all(const std::string& key) const { + std::vector a; + for (auto& e : entries) { + if (e.key == key) { + a.push_back(e.value); + } + } + return a; + } + std::string one(const std::string& key) const { + auto a = all(key); + if (a.size() != 1) { + throw RadioError("missing or repeated field: " + key); + } + return a[0]; + } + int integer(const std::string& key) const { + auto s = one(key); + std::size_t n; + int v = std::stoi(s, &n); + if (n != s.size()) { + throw RadioError("invalid integer: " + key); + } + return v; + } + double real(const std::string& key) const { + auto s = one(key); + std::size_t n; + double v = std::stod(s, &n); + if (n != s.size() || !std::isfinite(v)) { + throw RadioError("invalid number: " + key); + } + return v; + } + Doc section(std::size_t start, std::size_t end) const { + return {{entries.begin() + start, entries.begin() + end}}; + } +}; +std::string read(const std::filesystem::path& p) { + std::ifstream f(p); + if (!f) { + throw RadioError("cannot open job metadata: " + p.string()); + } + std::ostringstream s; + s << f.rdbuf(); + if (f.bad()) { + throw RadioError("job read failed"); + } + return s.str(); +} +Doc document(const std::filesystem::path& p) { + return {parse_job_text(read(p))}; +} +std::string upper(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { + return char(std::toupper(c)); + }); + return s; +} +bool boolean(std::string s) { + s = upper(s); + if (s == "TRUE" || s == "1" || s == "YES") { + return true; + } + if (s == "FALSE" || s == "0" || s == "NO") { + return false; + } + throw RadioError("invalid boolean"); +} +std::vector sections( + const Doc& doc, const std::vector& keys) { + std::vector starts; + for (auto& key : keys) { + std::size_t i = 0; + while (i < doc.entries.size() && doc.entries[i].key != key) { + ++i; + } + if (i == doc.entries.size() + || (!starts.empty() && i <= starts.back())) { + throw RadioError("missing or unordered section: " + key); + } + starts.push_back(i); + } + std::vector out; + for (std::size_t i = 0; i < starts.size(); ++i) { + out.push_back(doc.section( + starts[i], + i + 1 < starts.size() ? starts[i + 1] : doc.entries.size())); + } + return out; +} +J coeff(const Doc& d, const std::string& k, int occurrence, double scale) { + auto rows = d.all(k); + if (occurrence < 0 || std::size_t(occurrence) >= rows.size()) { + throw RadioError("missing polynomial: " + k); + } + std::istringstream in(rows[occurrence]); + J out = J::array(); + double x; + while (in >> x) { + x *= scale; + if (!std::isfinite(x)) { + throw RadioError("nonfinite model"); + } + out.push_back(x); + } + if (!in.eof() || out.empty()) { + throw RadioError("invalid model polynomial"); + } + return out; +} +double low(const J& f) { + return f.at("band_edge_hz").get() + - (f.at("sideband") == "LSB" ? f.at("bandwidth_hz").get() : 0.); +} +bool same_frequency(double a, double b) { + return std::abs(a - b) + <= std::max(1e-6, 1e-12 * std::max(std::abs(a), std::abs(b))); +} +struct Grid { + int channels; + double low, spacing; +}; +Grid grid(const J& frequency) { + const int channels = frequency.at("channel_count"); + return {channels, + low(frequency), + frequency.at("bandwidth_hz").get() / channels}; +} +bool same_grid(const Grid& a, const Grid& b) { + return a.channels == b.channels && same_frequency(a.low, b.low) + && same_frequency(a.spacing, b.spacing); +} +void validate_spectral_geometry(const J& job) { + const auto& frequencies = job.at("frequencies"); + const auto& streams = job.at("datastreams"); + auto band_grid = [&](int stream_id, int band_id) { + for (const auto& stream : streams) { + if (stream.at("identifier") != stream_id) { + continue; + } + for (const auto& band : stream.at("recorded_bands")) { + if (band.at("identifier") == band_id) { + return grid( + frequencies.at(band.at("frequency_id").get())); + } + } + for (const auto& zoom : stream.at("zoom_bands")) { + if (zoom.at("identifier") == band_id) { + return grid(frequencies.at( + zoom.at("output_frequency_id").get())); + } + } + } + throw RadioError("unknown visibility band"); + }; + for (const auto& stream : streams) { + const auto& bands = stream.at("recorded_bands"); + if (bands.empty()) { + throw RadioError("recording declares no bands"); + } + const int channels = + frequencies.at(bands.at(0).at("frequency_id").get()) + .at("channel_count"); + for (const auto& band : bands) { + const auto& frequency = + frequencies.at(band.at("frequency_id").get()); + const double rate = frequency.at("bandwidth_hz").get() + * (stream.at("sampling") == "real" ? 2. : 1.); + if (!same_frequency(rate, stream.at("sample_rate_hz")) + || frequency.at("channel_count") != channels) { + throw RadioError( + "native recorded bands must share one FFT grid"); + } + } + for (const auto& zoom : stream.at("zoom_bands")) { + const auto parent = + band_grid(stream.at("identifier"), zoom.at("parent_band_id")); + const auto output = + grid(frequencies.at(zoom.at("output_frequency_id").get())); + const int start = zoom.at("start_channel"); + if (start < 0 || output.channels > parent.channels - start + || !same_grid( + output, + {output.channels, + parent.low + start * parent.spacing, + parent.spacing})) { + throw RadioError( + "zoom output must describe its channel-aligned parent slice"); + } + } + } + for (const auto& baseline : job.at("baselines")) { + for (const auto& product : baseline.at("products")) { + const auto a = band_grid( + baseline.at("datastream_a_id"), product.at("band_a_id")); + const auto b = band_grid( + baseline.at("datastream_b_id"), product.at("band_b_id")); + const auto output = grid( + frequencies.at(product.at("output_frequency_id").get())); + if (!same_grid(a, b) || !same_grid(a, output)) { + throw RadioError( + "visibility bands require unsupported spectral regridding"); + } + } + } +} +void validate_fringe_order(const J& job) { + const int order = job.at("fringe_approximation_order"); + if (order < 0 || order > 2) { + throw RadioError("unsupported fringe approximation order"); + } + if (order != 2) { + return; + } + auto check = [](const J& coefficients) { + for (std::size_t i = 3; i < coefficients.size(); ++i) { + if (coefficients[i].get() != 0.) { + throw RadioError( + "fringe order 2 does not support cubic or higher delay terms"); + } + } + }; + for (const auto& station : job.at("stations")) { + check(station.at("clock").at("coefficients_s")); + } + for (const auto& scan : job.at("scans")) { + for (const auto& model : scan.at("station_models")) { + check(model.at("delay_coefficients_s")); + } + } +} +} // namespace +std::string load_interop_job_json(const std::string& name) { + namespace fs = std::filesystem; + auto path = fs::weakly_canonical(fs::absolute(name)); + auto input = document(path); + auto calcpath = + fs::weakly_canonical(path.parent_path() / input.one("CALC FILENAME")); + auto calc = document(calcpath); + auto im = document( + fs::weakly_canonical(calcpath.parent_path() / calc.one("IM FILENAME"))); + if (upper(input.one("OUTPUT FORMAT")) != "SWIN") { + throw RadioError("unsupported output format"); + } + const auto scans = calc.integer("NUM SCANS"), + stations = input.integer("TELESCOPE ENTRIES"); + if (scans < 1 || stations < 1 || im.integer("NUM SCANS") != scans) { + throw RadioError("invalid scan/station counts"); + } + const int mjd = input.integer("START MJD"); + const D seconds(input.one("START SECONDS")); + J job = {{"schema_version", 1}, + {"start_time", {{"mjd", mjd}, {"seconds_of_day", seconds.str()}}}, + {"execution_duration_s", D(input.one("EXECUTE TIME (SEC)")).str()}, + {"integration_duration_s", D(input.one("INT TIME (SEC)")).str()}, + {"fringe_approximation_order", input.integer("FRINGE ROTN ORDER")}, + {"write_autocorrelations", boolean(input.one("WRITE AUTOCORRS"))}, + {"output", {{"format", "swin"}, {"spectral_placement", 0}}}}; + auto sub = D(input.one("SUBINT NANOSECONDS")); + sub.scale += 9; + job["subintegration_duration_s"] = sub.str(); + job["buffered_reader"] = nullptr; + const std::vector> geometry = { + {"buffer_factor", "DATA BUFFER FACTOR"}, + {"data_segments", "NUM DATA SEGMENTS"}, + {"buffered_ffts", "NUM BUFFERED FFTS"}}; + for (auto& kv : geometry) { + if (!input.all(kv.second).empty()) { + job["buffered_reader"] = J::object(); + break; + } + } + if (!job["buffered_reader"].is_null()) { + for (auto& kv : geometry) { + int v = input.integer(kv.second); + if (v < 1) { + throw RadioError("invalid buffer geometry"); + } + job["buffered_reader"][kv.first] = v; + } + } + job["unsupported_features"] = { + {"pulsar_binning", boolean(input.one("PULSAR BINNING"))}, + {"phased_array_output", boolean(input.one("PHASED ARRAY"))}}; + for (const auto* key : {"multiple_phase_centres", + "phase_calibration", + "system_temperature_calibration", + "switched_power", + "polarization_conversion", + "output_band_stitching", + "spectral_kurtosis"}) { + job["unsupported_features"][key] = false; + } + if (job["unsupported_features"]["pulsar_binning"].get() + || job["unsupported_features"]["phased_array_output"].get()) { + throw RadioError("unsupported pulsar/phased-array output"); + } + job["stations"] = J::array(); + for (int i = 0; i < stations; ++i) { + auto id = std::to_string(i); + D epoch(input.one("CLOCK REF MJD " + id)); + I day = epoch.value / power(epoch.scale); + D sod = (epoch - D(day, 0)).times(86400); + J clock = J::array(); + int order = input.integer("CLOCK POLY ORDER " + id); + if (order < 0 || order > 100) { + throw RadioError("invalid clock order"); + } + for (int k = 0; k <= order; ++k) { + clock.push_back( + input.real("CLOCK COEFF " + id + "/" + std::to_string(k)) + * 1e-6); + } + job["stations"].push_back( + {{"identifier", i}, + {"name", input.one("TELESCOPE NAME " + id)}, + {"clock", + {{"epoch", + {{"mjd", static_cast(day)}, + {"seconds_of_day", sod.str()}}}, + {"coefficients_s", clock}}}}); + } + auto& frequencies = job["frequencies"] = J::array(); + for (int i = 0; i < input.integer("FREQ ENTRIES"); ++i) { + auto id = std::to_string(i); + auto side = upper(input.one("SIDEBAND " + id)); + if (side != "U" && side != "L") { + throw RadioError("invalid sideband"); + } + frequencies.push_back( + {{"identifier", i}, + {"band_edge_hz", input.real("FREQ (MHZ) " + id) * 1e6}, + {"bandwidth_hz", input.real("BW (MHZ) " + id) * 1e6}, + {"sideband", side == "U" ? "USB" : "LSB"}, + {"channel_count", input.integer("NUM CHANNELS " + id)}, + {"channels_to_average", input.integer("CHANS TO AVG " + id)}, + {"oversampling", input.integer("OVERSAMPLE FAC. " + id)}, + {"decimation", input.integer("DECIMATION FAC. " + id)}}); + const auto& frequency = frequencies.back(); + const int channels = frequency.at("channel_count"), + average = frequency.at("channels_to_average"); + if (frequency.at("bandwidth_hz").get() <= 0 + || frequency.at("band_edge_hz").get() < 0 || channels <= 0 + || average <= 0 || channels % average) { + throw RadioError("invalid native frequency geometry"); + } + if (frequency.at("oversampling") != 1 + || frequency.at("decimation") != 1) { + throw RadioError( + "native oversampling and decimation factors must be 1"); + } + } + std::vector starts; + for (std::size_t i = 0; i < input.entries.size(); ++i) { + if (input.entries[i].key == "TELESCOPE INDEX") { + starts.push_back(i); + } + } + if (starts.size() != std::size_t(input.integer("DATASTREAM ENTRIES"))) { + throw RadioError("datastream section count mismatch"); + } + J streams = J::array(); + for (std::size_t i = 0; i < starts.size(); ++i) { + auto d = input.section( + starts[i], + i + 1 < starts.size() ? starts[i + 1] : input.entries.size()); + auto id = std::to_string(i); + J bands = J::array(), zoom = J::array(), files = J::array(); + std::vector recfreq; + int count = 0; + for (int f = 0; f < d.integer("NUM RECORDED FREQS"); ++f) { + recfreq.push_back(d.integer("REC FREQ INDEX " + std::to_string(f))); + count += d.integer("NUM REC POLS " + std::to_string(f)); + } + auto format = upper(d.one("DATA FORMAT")); + int frame = d.integer("DATA FRAME SIZE"); + std::vector threads(count, 0); + if (format == "VDIF") { + if (frame == 4032 && count > 1) { + for (int k = 0; k < count; ++k) { + threads[k] = k; + } + } + } else { + std::smatch match; + if (!std::regex_match( + format, + match, + std::regex( + "INTERLACEDVDIF/([0-9]+(?::[0-9]+)*)/[0-9]+/[0-9]+"))) { + throw RadioError("unsupported VDIF format"); + } + std::stringstream t(match[1].str()); + std::string v; + threads.clear(); + while (std::getline(t, v, ':')) { + threads.push_back(std::stoi(v)); + } + if (threads.size() != std::size_t(count)) { + throw RadioError("thread count mismatch"); + } + } + for (int b = 0; b < count; ++b) { + auto k = std::to_string(b); + auto slot = d.integer("REC BAND " + k + " INDEX"); + if (slot < 0 || std::size_t(slot) >= recfreq.size()) { + throw RadioError("recorded frequency slot out of range"); + } + bands.push_back( + {{"identifier", b}, + {"frequency_id", recfreq[slot]}, + {"polarization", d.one("REC BAND " + k + " POL")}, + {"thread_id", threads[b]}, + {"clock_offset_s", + d.real("CLK OFFSET " + std::to_string(slot) + " (us)") + * 1e-6}, + {"lo_offset_hz", + d.real( + "FREQ OFFSET " + std::to_string(slot) + " (Hz)")}}); + } + int zcount = 0; + std::vector zfreq; + for (int f = 0; f < d.integer("NUM ZOOM FREQS"); ++f) { + zfreq.push_back(d.integer("ZOOM FREQ INDEX " + std::to_string(f))); + zcount += d.integer("NUM ZOOM POLS " + std::to_string(f)); + } + for (int z = 0; z < zcount; ++z) { + auto k = std::to_string(z); + int slot = d.integer("ZOOM BAND " + k + " INDEX"); + if (slot < 0 || std::size_t(slot) >= zfreq.size()) { + throw RadioError("zoom slot out of range"); + } + int output = zfreq[slot]; + auto& f = frequencies.at(output); + auto pol = d.one("ZOOM BAND " + k + " POL"); + int parent = -1, start = 0; + for (auto& b : bands) { + auto& p = frequencies.at(b.at("frequency_id").get()); + double width = p.at("bandwidth_hz").get(), + outwidth = f.at("bandwidth_hz").get(), + tol = std::max(width, outwidth) * 1e-12; + double raw = (low(f) - low(p)) + / (width / p.at("channel_count").get()); + if (b.at("polarization") == pol && low(f) >= low(p) - tol + && low(f) + outwidth <= low(p) + width + tol + && std::abs(raw - std::round(raw)) <= 1e-7) { + if (parent >= 0) { + throw RadioError("ambiguous zoom parent"); + } + parent = b.at("identifier"); + start = int(std::round(raw)); + } + } + if (parent < 0) { + throw RadioError("missing zoom parent"); + } + zoom.push_back( + {{"identifier", count + z}, + {"parent_band_id", parent}, + {"output_frequency_id", output}, + {"start_channel", start}, + {"channel_count", f.at("channel_count")}, + {"polarization", pol}}); + } + const int source_count = input.integer("D/STREAM " + id + " FILES"); + if (source_count < 0) { + throw RadioError("recording file count must be nonnegative"); + } + for (int f = 0; f < source_count; ++f) { + files.push_back( + fs::weakly_canonical( + path.parent_path() + / input.one("FILE " + id + "/" + std::to_string(f))) + .string()); + } + auto sampling = upper(d.one("DATA SAMPLING")); + if (sampling != "REAL" && sampling != "COMPLEX" + && sampling != "COMPLEX_DSB") { + throw RadioError("unsupported sampling"); + } + std::string kind = sampling == "REAL" ? "real" + : sampling == "COMPLEX" ? "complex" + : "complex_dsb"; + if (recfreq.empty()) { + throw RadioError("missing recorded frequency"); + } + streams.push_back( + {{"identifier", i}, + {"station_id", d.integer("TELESCOPE INDEX")}, + {"sampling", kind}, + {"quantization_bits", d.integer("QUANTISATION BITS")}, + {"frame_size_bytes", frame}, + {"source_files", files}, + {"recorded_bands", bands}, + {"zoom_bands", zoom}, + {"sample_rate_hz", + frequencies.at(recfreq[0]).at("bandwidth_hz").get() + * (sampling == "REAL" ? 2. : 1.)}, + {"data_format", "VDIF"}, + {"data_source", "file"}}); + } + job["baselines"] = J::array(); + std::vector keys; + for (int b = 0; b < input.integer("BASELINE ENTRIES"); ++b) { + keys.push_back("D/STREAM A INDEX " + std::to_string(b)); + } + auto bs = sections(input, keys); + for (std::size_t b = 0; b < bs.size(); ++b) { + auto id = std::to_string(b); + auto& d = bs[b]; + int a = d.integer("D/STREAM A INDEX " + id), + c = d.integer("D/STREAM B INDEX " + id); + auto& sa = streams.at(a); + streams.at(c); + J products = J::array(); + std::set> product_keys; + std::map occurrences; + for (int f = 0; f < d.integer("NUM FREQS " + id); ++f) { + auto fid = id + "/" + std::to_string(f); + auto targets = d.all("TARGET FREQ " + fid); + if (targets.size() > 1) { + throw RadioError("repeated target frequency"); + } + for (int p = 0; p < d.integer("POL PRODUCTS " + fid); ++p) { + auto left = d.all("D/STREAM A BAND " + std::to_string(p)), + right = d.all("D/STREAM B BAND " + std::to_string(p)); + auto n = occurrences[p]++; + int ba = std::stoi(left.at(n)), bb = std::stoi(right.at(n)), + output = -1; + if (!targets.empty()) { + output = std::stoi(targets[0]); + } else { + for (auto& band : sa["recorded_bands"]) { + if (band.at("identifier") == ba) { + output = band.at("frequency_id"); + } + } + for (auto& band : sa["zoom_bands"]) { + if (band.at("identifier") == ba) { + output = band.at("output_frequency_id"); + } + } + } + if (output < 0) { + throw RadioError("invalid product band"); + } + if (!product_keys.emplace(ba, bb, output).second) { + throw RadioError("duplicate baseline product"); + } + products.push_back( + {{"band_a_id", ba}, + {"band_b_id", bb}, + {"output_frequency_id", output}}); + } + } + if (!streams.at(a).at("source_files").empty() + && !streams.at(c).at("source_files").empty()) { + job["baselines"].push_back( + {{"identifier", b}, + {"datastream_a_id", a}, + {"datastream_b_id", c}, + {"products", products}}); + } + } + job["datastreams"] = J::array(); + for (auto& s : streams) { + if (!s.at("source_files").empty()) { + job["datastreams"].push_back(s); + } + } + keys.clear(); + for (int s = 0; s < scans; ++s) { + keys.push_back("SCAN " + std::to_string(s) + " POINTING SRC"); + } + auto ims = sections(im, keys); + job["scans"] = J::array(); + std::set scan_names; + D fraction(seconds.value % power(seconds.scale), seconds.scale); + for (int s = 0; s < scans; ++s) { + auto sid = std::to_string(s), prefix = "SCAN " + sid; + const auto name = calc.one(prefix + " IDENTIFIER"); + if (!scan_names.insert(name).second) { + throw RadioError("duplicate scan identifier: " + name); + } + const D start = D(calc.one(prefix + " START (S)")) - fraction; + const D effective_start = start.value < 0 ? D(0, 0) : start; + auto& d = ims[s]; + if (calc.integer(prefix + " NUM PHS CTRS") != 1) { + throw RadioError("multiple phase centres unsupported"); + } + J models = J::array(); + int polys = d.integer(prefix + " NUM POLY"); + if (polys < 1) { + throw RadioError("missing scan polynomials"); + } + D first_epoch(0, 0); + std::set runtime_epochs; + for (int p = 0; p < polys; ++p) { + auto poly = prefix + " POLY " + std::to_string(p); + D epoch = + D(std::to_string((d.integer(poly + " MJD") - mjd) * 86400)) + + D(d.one(poly + " SEC")) - seconds; + const auto runtime_epoch = std::stod(epoch.str()); + if (!std::isfinite(runtime_epoch) + || !runtime_epochs.insert(runtime_epoch).second) { + throw RadioError( + "duplicate/nonfinite model epoch at runtime precision"); + } + if (p == 0 || (epoch - first_epoch).value < 0) { + first_epoch = epoch; + } + for (int a = 0; a < stations; ++a) { + auto key = "SRC 0 ANT " + std::to_string(a); + models.push_back( + {{"station_id", a}, + {"epoch_offset_s", epoch.str()}, + {"delay_coefficients_s", + coeff(d, key + " DELAY (us)", p, 1e-6)}, + {"u_coefficients_m", coeff(d, key + " U (m)", p, 1.)}, + {"v_coefficients_m", coeff(d, key + " V (m)", p, 1.)}, + {"w_coefficients_m", coeff(d, key + " W (m)", p, 1.)}}); + } + } + if ((first_epoch - effective_start).value > 0) { + throw RadioError( + "first model starts after the scan execution interval"); + } + job["scans"].push_back( + {{"identifier", name}, + {"start_offset_s", start.str()}, + {"duration_s", D(calc.one(prefix + " DUR (S)")).str()}, + {"source_id", calc.one(prefix + " POINTING SRC")}, + {"configuration_id", "0"}, + {"station_models", models}}); + } + validate_spectral_geometry(job); + validate_fringe_order(job); + return job.dump(); +} +} // namespace xbeam + +namespace xbeam { +namespace { +const J& by_id(const J& rows, int id) { + for (const auto& row : rows) { + if (row.at("identifier") == id) { + return row; + } + } + throw RadioError("unknown native job identifier"); +} +double polynomial(const J& coefficients, double x) { + double result = 0; + for (auto it = coefficients.rbegin(); it != coefficients.rend(); ++it) { + result = result * x + it->get(); + } + return result; +} +std::int64_t nearest(double x) { + if (!std::isfinite(x) || x < -std::ldexp(1., 63) + || x >= std::ldexp(1., 63)) { + throw RadioError("sample coordinate exceeds int64"); + } + return static_cast(std::nearbyint(x)); +} +D absolute_time(const J& time) { + return D(time.at("seconds_of_day").get()) + + D(std::to_string(time.at("mjd").get())).times(86400); +} +} // namespace +struct RadioJob::Impl { + struct Clock { + double epoch; + std::vector coefficients; + }; + struct Model { + double epoch; + std::vector coefficients; + }; + struct Band { + int station; + std::int64_t fft; + double rate, offset; + }; + J job; + std::map clocks; + std::map, Band> bands; + std::map, std::vector> models; + int fringe_order; + explicit Impl(const std::string& path) + : job(J::parse(load_interop_job_json(path))) { + fringe_order = job.at("fringe_approximation_order"); + if (fringe_order < 0 || fringe_order > 2) { + throw RadioError("unsupported fringe approximation order"); + } + auto coefficients = [](const J& array) { + auto values = array.get>(); + for (double value : values) { + if (!std::isfinite(value)) { + throw RadioError("nonfinite native model coefficient"); + } + } + return values; + }; + for (const auto& station : job.at("stations")) { + double epoch = std::stod( + (absolute_time(job.at("start_time")) + - absolute_time(station.at("clock").at("epoch"))) + .str()); + clocks.emplace( + station.at("identifier"), + Clock{epoch, + coefficients(station.at("clock").at("coefficients_s"))}); + } + for (const auto& stream : job.at("datastreams")) { + for (const auto& band : stream.at("recorded_bands")) { + const auto& frequency = + by_id(job.at("frequencies"), band.at("frequency_id")); + auto channels = + frequency.at("channel_count").get(); + int components = stream.at("sampling") == "real" ? 2 : 1; + if (channels <= 0 || channels > INT64_MAX / components) { + throw RadioError("invalid native FFT geometry"); + } + bands.emplace( + std::make_pair( + stream.at("identifier").get(), + band.at("identifier").get()), + Band{stream.at("station_id"), + channels * components, + stream.at("sample_rate_hz"), + band.at("clock_offset_s")}); + } + } + for (const auto& scan : job.at("scans")) { + for (const auto& model : scan.at("station_models")) { + models[{scan.at("identifier"), model.at("station_id")}] + .push_back( + {std::stod( + model.at("epoch_offset_s").get()), + coefficients(model.at("delay_coefficients_s"))}); + } + } + for (auto& entry : models) { + std::stable_sort( + entry.second.begin(), + entry.second.end(), + [](const Model& a, const Model& b) { + return a.epoch < b.epoch; + }); + } + } +}; +RadioJob::RadioJob(const std::string& input) + : impl(std::make_unique(input)) {} +RadioJob::~RadioJob() = default; +std::string RadioJob::normalized_json() const { + return impl->job.dump(); +} +std::vector RadioJob::source_blocks( + const std::string& scan_name, + int stream_id, + int band_id, + const double* starts, + std::size_t count, + bool literal_endpoint, + bool phase_parameters) const { + auto band_it = impl->bands.find({stream_id, band_id}); + if (band_it == impl->bands.end()) { + throw RadioError("unknown native band"); + } + const auto& band = band_it->second; + auto model_it = impl->models.find({scan_name, band.station}); + if (model_it == impl->models.end() || model_it->second.empty()) { + throw RadioError("missing station model"); + } + const auto& models = model_it->second; + const auto& clock = impl->clocks.at(band.station); + const auto fft = band.fft; + const double rate = band.rate, clock_epoch = clock.epoch, + band_offset = band.offset; + if (!std::isfinite(rate) || rate <= 0) { + throw RadioError("invalid native sample rate"); + } + auto evaluate = [](const std::vector& coefficients, double x) { + double value = 0; + for (auto it = coefficients.rbegin(); it != coefficients.rend(); ++it) { + value = value * x + *it; + } + return value; + }; + std::vector out; + out.reserve(count); + for (std::size_t i = 0; i < count; ++i) { + if (!std::isfinite(starts[i]) || (i && starts[i] < starts[i - 1])) { + throw RadioError("block starts must be finite and ordered"); + } + const double centre = literal_endpoint + ? starts[i] + : starts[i] + double(fft - 1) / (2. * rate); + const Impl::Model* selected = nullptr; + if (models.size() == 1) { + selected = &models[0]; + } else { + auto position = std::upper_bound( + models.begin(), + models.end(), + centre, + [](double t, const Impl::Model& m) { return t < m.epoch; }); + if (position != models.begin()) { + selected = &*(position - 1); + } + } + if (!selected) { + throw RadioError("station model does not cover block"); + } + double delay = + evaluate(selected->coefficients, centre - selected->epoch); + delay += evaluate(clock.coefficients, centre + clock_epoch); + delay += band_offset; + out.push_back( + {fft, nearest(starts[i] * rate), nearest(delay * rate), delay}); + auto& block = out.back(); + block.block_start = starts[i]; + if (!phase_parameters) { + continue; + } + auto value_at = [&](double t) { + double value = + evaluate(selected->coefficients, t - selected->epoch); + value += evaluate(clock.coefficients, t + clock_epoch); + return value + band_offset; + }; + const int order = impl->fringe_order; + if (order == 0 || fft == 1) { + block.delay_zero = delay; + } else { + block.delay_zero = value_at(starts[i]); + double right = value_at(starts[i] + double(fft - 1) / rate); + if (order == 1) { + block.delay_step = (right - block.delay_zero) / double(fft - 1); + } else { + const double last = double(fft - 1), middle_index = last / 2.; + const double middle = value_at(starts[i] + middle_index / rate); + block.delay_quadratic = 2. + * (right + block.delay_zero - 2. * middle) / (last * last); + block.delay_step = + (middle - block.delay_zero + - block.delay_quadratic * middle_index * middle_index) + / middle_index; + } + } + block.residual = double(block.coarse_sample) / rate - delay; + } + return out; +} +} // namespace xbeam + +#include "xbeam/replay.hpp" +namespace xbeam { +JobRequestInputs RadioJob::requests( + const std::string& scan_name, + int stream_id, + int band_id, + const RequestGeometry& g, + std::int64_t first, + std::int64_t stop, + std::int64_t timeline_count, + std::int64_t job_slot, + std::int64_t origin_slot, + bool mean, + std::int64_t limit) const { + if (first < 0 || first >= stop || stop > timeline_count || limit < 1 + || origin_slot != job_slot) { + throw RadioError("invalid request scope or timestamp origin"); + } + const auto& stream = by_id(impl->job.at("datastreams"), stream_id); + const double rate = stream.at("sample_rate_hz"); + request_cursor(0, 0, g, rate); + const auto width = g.ffts_per_request; + if (timeline_count > INT64_MAX - width) { + throw RadioError("FFT timeline exceeds int64"); + } + const auto start = first / width * width; + const auto rows = (stop - start) / width + ((stop - start) % width != 0); + if (rows > limit || I(rows) * width > 16777216) { + throw RadioError("request scope exceeds memory budget"); + } + const J* scan = nullptr; + for (const auto& s : impl->job.at("scans")) { + if (s.at("identifier") == scan_name) { + scan = &s; + } + } + if (!scan) { + throw RadioError("unknown request scan"); + } + const double scan_origin = + std::max(0., std::stod(scan->at("start_offset_s").get())); + const auto total = std::size_t(rows) * width; + std::vector times(total), left_times(rows), right_times(rows), + literal_right; + const double stride = double(g.fft_samples) / rate, + centre_offset = double(g.fft_samples - 1) / (2. * rate); + for (std::int64_t row = 0; row < rows; ++row) { + auto index = start + row * width; + double t = scan_origin + double(index) * stride; + left_times[row] = t - centre_offset; + right_times[row] = + t + double(width) * double(g.fft_samples) / rate - centre_offset; + for (std::int64_t column = 0; column < width; ++column) { + times[std::size_t(row) * width + column] = + scan_origin + double(index + column) * stride; + } + } + const auto ordinary = source_blocks( + scan_name, stream_id, band_id, times.data(), total, false, false); + const auto left = source_blocks( + scan_name, stream_id, band_id, left_times.data(), rows, false, false); + const auto right = source_blocks( + scan_name, stream_id, band_id, right_times.data(), rows, false, false); + std::vector literal_left, literal_stop; + if (mean) { + literal_right.resize(total); + for (std::size_t i = 0; i < total; ++i) { + literal_right[i] = times[i] + stride; + } + literal_left = source_blocks( + scan_name, stream_id, band_id, times.data(), total, true, false); + literal_stop = source_blocks( + scan_name, + stream_id, + band_id, + literal_right.data(), + total, + true, + false); + } + JobRequestInputs out{ + first, stop, timeline_count, origin_slot, rate, {}, {}, {}, {}, {}}; + auto checked = [](I value) { + if (value < INT64_MIN || value > INT64_MAX) { + throw RadioError("request sample coordinate exceeds int64"); + } + return static_cast(value); + }; + auto offset = [&](double value) { + value = std::trunc(value + .5); + if (!std::isfinite(value) || value < -std::ldexp(1., 63) + || value >= std::ldexp(1., 63)) { + throw RadioError("request clock exceeds int64"); + } + return static_cast(value); + }; + for (std::int64_t row = 0; row < rows; ++row) { + RequestFact fact; + fact.first_fft = start + row * width; + fact.active_count = std::min(width, timeline_count - fact.first_fft); + const double t = scan_origin + double(fact.first_fft) * stride; + fact.desired_sample = checked( + I(offset(t * rate)) - offset(left[row].centre_delay * rate)); + fact.endpoint_drift = right[row].centre_delay - left[row].centre_delay; + std::vector outputs; + std::vector centres, averages; + for (std::int64_t column = 0; column < width; ++column) { + auto i = std::size_t(row) * width + column; + const auto& block = ordinary[i]; + if (block.fft_samples != g.fft_samples) { + throw RadioError("job and request FFT lengths differ"); + } + fact.ordinary_samples.push_back( + checked(I(block.output_sample) - block.coarse_sample)); + outputs.push_back(block.output_sample); + centres.push_back(block.centre_delay); + if (mean) { + double value = literal_left[i].centre_delay * .5 + + literal_stop[i].centre_delay * .5; + if (!std::isfinite(value)) { + throw RadioError("nonfinite read average"); + } + averages.push_back(value); + } + } + out.requests.push_back(std::move(fact)); + out.output_samples.push_back(std::move(outputs)); + out.centre_delays.push_back(std::move(centres)); + out.read_average.push_back(std::move(averages)); + out.left_delays.push_back(left[row].centre_delay); + } + return out; +} +} // namespace xbeam + +#include "xbeam/window.hpp" +namespace xbeam { +std::int64_t RadioJob::frame_slot(std::int64_t fps, int epoch) const { + if (fps <= 0 || fps > (1 << 24)) { + throw RadioError("invalid VDIF frame rate"); + } + const auto& time = impl->job.at("start_time"); + D seconds = D(I(time.at("mjd").get()) * 86400, 0) + + D(time.at("seconds_of_day")); + auto value = seconds.times(fps); + if (value.value % power(value.scale)) { + throw RadioError("job start must be VDIF-frame aligned"); + } + // The epoch's leap contribution is distinct from elapsed header seconds. + int leaps = 0; + for (int t : {12, 18, 25, 31, 34}) { + leaps += epoch >= t; + } + reference_epoch_seconds(epoch); + I slot = + value.value / power(value.scale) - (I(51544) * 86400 - leaps) * fps; + if (slot < INT64_MIN || slot > INT64_MAX) { + throw RadioError("job frame slot exceeds int64"); + } + return static_cast(slot); +} +std::pair RadioJob::subintegration_ratio() const { + D value(impl->job.at("subintegration_duration_s")); + while (value.scale && value.value % 10 == 0) { + value.value /= 10; + --value.scale; + } + if (value.value <= 0 || value.value > INT64_MAX + || power(value.scale) > INT64_MAX) { + throw RadioError("subintegration rational exceeds int64"); + } + return {static_cast(value.value), + static_cast(power(value.scale))}; +} +} // namespace xbeam +namespace xbeam { +SwinMetadata RadioJob::record_metadata( + const std::string& scan_name, + std::int64_t integration, + int a, + int b, + int frequency, + char pa, + char pb) const { + if (integration < 0) { + throw RadioError("negative output integration"); + } + const J* scan = nullptr; + for (const auto& s : impl->job.at("scans")) { + if (s.at("identifier") == scan_name) { + scan = &s; + } + } + if (!scan) { + throw RadioError("unknown output scan"); + } + D origin(scan->at("start_offset_s")); + if (origin.value < 0) { + origin = D(0, 0); + } + D duration(impl->job.at("integration_duration_s")); + D midpoint = duration.times(I(integration) * 2 + 1).times(5); + midpoint.scale += 1; + D time = origin + midpoint; + D seconds = D(impl->job.at("start_time").at("seconds_of_day")) + time; + I denominator = power(seconds.scale), + days = seconds.value / (denominator * 86400); + if (seconds.value % (denominator * 86400) < 0) { + --days; + } + D within = seconds - D(days * 86400, 0); + I day = impl->job.at("start_time").at("mjd").get() + days; + if (day < 0 || day > UINT32_MAX) { + throw RadioError("SWIN timestamp exceeds day range"); + } + int ai = -1, bi = -1; + const auto& stations = impl->job.at("stations"); + for (std::size_t k = 0; k < stations.size(); ++k) { + if (stations[k].at("identifier") == a) { + ai = k + 1; + } + if (stations[k].at("identifier") == b) { + bi = k + 1; + } + } + if (ai < 1 || bi < 1 || ai > 255 || bi > 255) { + throw RadioError("SWIN station index is unsupported"); + } + SwinMetadata m{}; + m.baseline = ai * 256 + bi; + m.mjd = static_cast(day); + m.seconds = std::stod(within.str()); + m.frequency_id = frequency; + m.polarization_a = pa; + m.polarization_b = pb; + m.configuration_id = + std::stoi(scan->at("configuration_id").get()); + m.source_id = std::stoi(scan->at("source_id").get()); + const double t = std::stod(time.str()); + auto coordinates = [&](int station, const char* field) { + const J* selected = nullptr; + double epoch = -std::numeric_limits::infinity(); + for (const auto& model : scan->at("station_models")) { + if (model.at("station_id") == station) { + double current = + std::stod(model.at("epoch_offset_s").get()); + if (current <= t && current >= epoch) { + selected = &model; + epoch = current; + } + } + } + if (!selected) { + throw RadioError("no coordinate model covers integration centre"); + } + return polynomial(selected->at(field), t - epoch); + }; + m.u = + coordinates(a, "u_coefficients_m") - coordinates(b, "u_coefficients_m"); + m.v = + coordinates(a, "v_coefficients_m") - coordinates(b, "v_coefficients_m"); + m.w = + coordinates(a, "w_coefficients_m") - coordinates(b, "w_coefficients_m"); + return m; +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/src/job_text.cpp b/src/cuphoton/xbeam/native/src/job_text.cpp new file mode 100644 index 00000000..f8c3a201 --- /dev/null +++ b/src/cuphoton/xbeam/native/src/job_text.cpp @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/radio.hpp" +#include +namespace xbeam { +namespace { +std::string trim(const std::string& s) { + const auto start = s.find_first_not_of(" \t\r\n\v\f"); + if (start == std::string::npos) { + return {}; + } + const auto end = s.find_last_not_of(" \t\r\n\v\f"); + return s.substr(start, end - start + 1); +} +} // namespace +std::vector parse_job_text(const std::string& text) { + std::vector entries; + std::istringstream stream(text); + std::string line; + std::size_t number = 0; + while (std::getline(stream, line)) { + ++number; + const auto stripped = trim(line); + if (stripped.empty() || stripped[0] == '#' || stripped[0] == '@') { + continue; + } + const auto colon = line.find(':'); + if (colon == std::string::npos) { + throw RadioError( + "job line " + std::to_string(number) + ": expected KEY: value"); + } + auto key = trim(line.substr(0, colon)); + if (key.empty()) { + throw RadioError( + "job line " + std::to_string(number) + ": empty field name"); + } + entries.push_back( + {std::move(key), trim(line.substr(colon + 1)), number}); + } + return entries; +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/src/main.cpp b/src/cuphoton/xbeam/native/src/main.cpp new file mode 100644 index 00000000..a547b380 --- /dev/null +++ b/src/cuphoton/xbeam/native/src/main.cpp @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/radio.hpp" +#include "xbeam/recording.hpp" +#include +#include +int main(int argc, char** argv) { + if (argc == 10 && std::string(argv[1]) == "inspect-replay") { + try { + xbeam::RadioJob job(argv[2]); + xbeam::RadioRecording recording(job, std::stoi(argv[3]), argv[5]); + auto plan = recording.plan( + argv[6], + std::stoi(argv[4]), + std::stoll(argv[7]), + std::stoll(argv[8]), + std::stoll(argv[9])); + const auto& g = recording.geometry(); + std::cout << "{\"job_slot\":" << g.job_slot + << ",\"allocation_frames\":" + << g.windows.allocation_frames << ",\"rows\":["; + bool comma = false; + for (const auto& r : plan.replay) { + if (comma) { + std::cout << ','; + } + comma = true; + std::cout << '[' << r.fft << ',' << r.physical_start << ',' + << r.timing_displacement << ',' << r.valid_first + << ',' << r.valid_stop << ',' << int(r.accepted) + << ']'; + } + std::cout << "]}\n"; + return 0; + } catch (const std::exception& e) { + std::cerr << "ERROR: " << e.what() << '\n'; + return 1; + } + } + + if (argc == 4 && std::string(argv[1]) == "inspect-index") { + try { + xbeam::IndexReader reader(argv[2], argv[3]); + std::cout << reader.metadata_json() << '\n'; + return 0; + } catch (const std::exception& e) { + std::cerr << "ERROR: " << e.what() << '\n'; + return 1; + } + } + + if (argc != 3 + || (std::string(argv[1]) != "inspect-vdif" + && std::string(argv[1]) != "inspect-job")) { + std::cerr + << "usage: xbeam-native inspect-vdif RECORDING.vdif | inspect-job " + "JOB.input\n"; + return 2; + } + try { + std::cout + << (std::string(argv[1]) == "inspect-job" + ? xbeam::load_interop_job_json(argv[2]) + : xbeam::inspect_vdif_json(argv[2])) + << '\n'; + return 0; + } catch (const std::exception& e) { + std::cerr << "ERROR: " << e.what() << '\n'; + return 1; + } +} diff --git a/src/cuphoton/xbeam/native/src/recording.cpp b/src/cuphoton/xbeam/native/src/recording.cpp new file mode 100644 index 00000000..5f740a72 --- /dev/null +++ b/src/cuphoton/xbeam/native/src/recording.cpp @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/recording.hpp" +#include +#include +#include +namespace xbeam { +namespace { +using J = nlohmann::json; +J recording_document(const RadioJob& job) { + auto document = J::parse(job.normalized_json()); + if (document.at("buffered_reader").is_null()) { + throw RadioError( + "native replay requires DATA BUFFER FACTOR, NUM DATA SEGMENTS " + "and NUM BUFFERED FFTS in the job metadata"); + } + return document; +} +const J& by_id(const J& array, int id) { + for (const auto& row : array) { + if (row.at("identifier") == id) { + return row; + } + } + throw RadioError("unknown native recording identifier"); +} +std::int64_t checked(__int128 value) { + if (value < INT64_MIN || value > INT64_MAX) { + throw RadioError("recording coordinate exceeds int64"); + } + return static_cast(value); +} +std::string source_path(const J& stream) { + if (stream.at("source_files").size() != 1) { + throw RadioError("raw replay requires one physical source per stream"); + } + return stream.at("source_files").at(0); +} +} // namespace +struct RadioRecording::Impl { + const RadioJob& job; + int stream_id; + J document, stream, metadata; + IndexReader index; + RecordingGeometry g{}; + std::unique_ptr timeline; + RawRequestState state; + Impl( + const RadioJob& input, + int id, + const std::string& sidecar, + IndexReader* validated) + : job(input), + stream_id(id), + document(recording_document(input)), + stream(by_id(document.at("datastreams"), id)), + index( + validated ? std::move(*validated) + : IndexReader(source_path(stream), sidecar)) { + metadata = J::parse(index.metadata_json()); + if (metadata.at("threads").size() != 1 + || stream.at("sampling") != "real") { + throw RadioError( + "buffered raw replay requires one real-sampled physical thread"); + } + const auto& thread = metadata.at("threads").at(0); + if (thread.at("complex_data").get() + || thread.at("bits_per_sample") != stream.at("quantization_bits") + || thread.at("frame_size_bytes") != stream.at("frame_size_bytes")) { + throw RadioError("sidecar and recording sample geometry differ"); + } + g.thread_id = thread.at("thread_id"); + g.channel_count = thread.at("channel_count"); + g.samples_per_frame = thread.at("samples_per_frame"); + g.frame_bytes = stream.at("frame_size_bytes"); + g.payload_bytes = g.frame_bytes - 32; + g.frame_count = metadata.at("frame_count"); + double rate = stream.at("sample_rate_hz"); + if (!std::isfinite(rate) || rate <= 0 || rate >= std::ldexp(1., 63) + || rate != std::trunc(rate)) { + throw RadioError( + "raw replay requires an integral positive sample rate"); + } + g.sample_rate = static_cast(rate); + if (g.samples_per_frame <= 0 || g.sample_rate % g.samples_per_frame + || g.channel_count <= 0 || stream.at("quantization_bits") != 2) { + throw RadioError("inconsistent native recording geometry"); + } + g.frames_per_second = g.sample_rate / g.samples_per_frame; + auto packed = index.physical_runs(); + if (packed.empty()) { + throw RadioError("physical timestamp history is empty"); + } + g.job_slot = job.frame_slot( + g.frames_per_second, packed.front().packed_timestamp >> 54); + const auto& bands = stream.at("recorded_bands"); + if (bands.empty()) { + throw RadioError("recording declares no bands"); + } + int logical = bands.at(0).at("thread_id"); + for (const auto& band : bands) { + if (band.at("thread_id") != logical) { + throw RadioError( + "single physical recording cannot alias multiple " + "requested threads"); + } + } + if (bands.size() != std::size_t(g.channel_count)) { + throw RadioError( + "recorded band count differs from packed channels"); + } + auto rational = job.subintegration_ratio(); + auto fft = checked( + __int128(by_id( + document.at("frequencies"), bands.at(0).at("frequency_id")) + .at("channel_count") + .get()) + * 2); + const auto& reader = document.at("buffered_reader"); + g.windows = size_windows( + g.samples_per_frame, + fft, + g.sample_rate, + rational.first, + rational.second, + reader.at("buffer_factor"), + reader.at("data_segments"), + reader.at("buffered_ffts"), + 0); + auto den = checked(__int128(2) * g.channel_count); + // Reduce the packing ratio before cursor divisibility checks. + std::int64_t num = 8; + auto a = num, b = den; + while (b) { + auto r = a % b; + a = b; + b = r; + } + num /= a; + den /= a; + g.request = {num, den, fft, g.windows.ffts_per_request, 8192, 4}; + g.extent = normal_extent( + g.frame_bytes, + g.payload_bytes, + g.windows.request_frames, + g.windows.read_buffer_ratio, + num, + den); + std::vector runs; + for (const auto& run : packed) { + runs.push_back( + {checked(run.first), + checked(run.stop), + packed_frame_slot( + run.packed_timestamp, g.frames_per_second)}); + } + timeline = std::make_unique( + runs, g.windows.allocation_frames); + state = + raw_request_state(*timeline, g.samples_per_frame, g.job_slot, true); + } +}; +RadioRecording::RadioRecording( + const RadioJob& job, int stream, const std::string& index) + : impl(std::make_unique(job, stream, index, nullptr)) {} +RadioRecording::RadioRecording( + const RadioJob& job, int stream, IndexReader&& index) + : impl(std::make_unique(job, stream, std::string(), &index)) {} +RadioRecording::~RadioRecording() = default; +const RecordingGeometry& RadioRecording::geometry() const { + return impl->g; +} +RadioReplayBatch RadioRecording::plan( + const std::string& scan, + int band, + std::int64_t first, + std::int64_t stop, + std::int64_t count) const { + const auto& g = impl->g; + auto requests = impl->job.requests( + scan, + impl->stream_id, + band, + g.request, + first, + stop, + count, + g.job_slot, + g.job_slot, + false, + 256); + auto windows = returned_windows( + *impl->timeline, + impl->state, + requests, + g.request, + g.extent, + 30, + 1, + checked(__int128(g.frame_count) * g.samples_per_frame)); + auto rows = compile_replay_rows( + windows, + g.request.fft_samples, + g.sample_rate, + checked(__int128(g.frame_count) * g.samples_per_frame)); + const J* selected_scan = nullptr; + for (const auto& s : impl->document.at("scans")) { + if (s.at("identifier") == scan) { + selected_scan = &s; + } + } + if (!selected_scan) { + throw RadioError("unknown recording scan"); + } + double origin = std::max( + 0., std::stod(selected_scan->at("start_offset_s").get())); + std::vector starts; + for (const auto& row : rows) { + starts.push_back( + origin + + double(row.fft) + * (double(g.request.fft_samples) / double(g.sample_rate))); + } + auto blocks = impl->job.source_blocks( + scan, impl->stream_id, band, starts.data(), starts.size()); + int channel = -1; + const auto& bands = impl->stream.at("recorded_bands"); + for (std::size_t k = 0; k < bands.size(); ++k) { + if (bands[k].at("identifier") == band) { + channel = k; + } + } + if (channel < 0) { + throw RadioError("unknown packed channel"); + } + return {std::move(blocks), std::move(rows), channel}; +} + +} // namespace xbeam + +namespace xbeam { +std::vector RadioRecording::frame_validity( + std::int64_t first, std::int64_t count) const { + if (first < 0 || count < 0) { + throw RadioError("negative physical validity scope"); + } + return impl->index.frame_validity(first, count); +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/src/replay.cpp b/src/cuphoton/xbeam/native/src/replay.cpp new file mode 100644 index 00000000..ef223e0d --- /dev/null +++ b/src/cuphoton/xbeam/native/src/replay.cpp @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/replay.hpp" +#include +#include +namespace xbeam { +namespace { +using Wide = __int128; +std::int64_t narrow(Wide x) { + if (x < INT64_MIN || x > INT64_MAX) { + throw RadioError("replay coordinate exceeds int64"); + } + return static_cast(x); +} +} // namespace +RawRequestState raw_request_state( + const PhysicalTimeline& timeline, + std::int64_t samples, + std::int64_t origin, + bool clock_owner) { + if (samples <= 0) { + throw RadioError("samples per frame must be positive"); + } + RawRequestState state; + state.allocation_frames = timeline.allocation_frames(); + state.samples_per_frame = samples; + state.timestamp_origin = origin; + for (auto anchor : timeline.anchors()) { + state.anchors.push_back(narrow((Wide(anchor) - origin) * samples)); + } + state.events.resize(state.anchors.size(), 0); + const auto& runs = timeline.physical_runs(); + for (std::size_t i = 1; i < runs.size(); ++i) { + Wide residual = Wide(runs[i].timestamp) - runs[i - 1].timestamp + - (runs[i - 1].stop - runs[i - 1].first) + - (runs[i].first - runs[i - 1].stop); + if (residual > 0) { + auto preceding = clock_owner ? runs[i - 1].stop : runs[i].first; + auto event = (preceding - 1) / state.allocation_frames; + if (event < 0 || std::size_t(event) >= state.events.size()) { + throw RadioError("event allocation exceeds timeline"); + } + state.events[event] = 1; + } + } + return state; +} +DiscardBatch compile_discards( + const RawRequestState& state, + const std::vector& facts, + const RequestGeometry& g, + double rate, + std::int64_t frame_bytes, + std::int64_t control_bits) { + if (frame_bytes <= 0 || control_bits <= 0 || state.allocation_frames <= 0 + || state.anchors.empty() + || state.anchors.size() != state.events.size()) { + throw RadioError("invalid framed replay geometry"); + } + const auto extent = narrow(Wide(state.allocation_frames) * frame_bytes); + request_cursor( + 0, 0, g, rate); // Geometry is validated even for an empty batch. + const auto bytes = + narrow(Wide(g.fft_samples) * g.ratio_denominator / g.ratio_numerator); + DiscardBatch out; + std::vector excluded; + for (const auto& fact : facts) { + const auto width = fact.ordinary_samples.size(); + const auto active = + fact.active_count ? std::size_t(fact.active_count) : width; + if (active < 1 || active > width) { + throw RadioError("invalid request count"); + } + if (fact.first_fft < 0 || width == 0 + || width > std::size_t(g.ffts_per_request)) { + throw RadioError("invalid replay request width/start"); + } + narrow(Wide(fact.first_fft) + width - 1); + auto selected = + std::upper_bound( + state.anchors.begin(), state.anchors.end(), fact.desired_sample) + - state.anchors.begin() - 1; + bool allocated = + selected >= 0 && std::size_t(selected) < state.anchors.size(); + auto local = allocated + ? narrow(Wide(fact.desired_sample) - state.anchors[selected]) + : 0; + auto cursor = request_cursor(local, fact.endpoint_drift, g, rate); + bool valid = allocated && cursor.valid; + if (!valid) { + cursor = {}; + } + bool event = valid && state.events[selected]; + std::vector mask(width, valid ? 1 : 0); + if (event) { + bool any = false; + for (std::size_t j = 0; j < width; ++j) { + mask[j] = + narrow(Wide(cursor.origin_bytes) + Wide(j) * bytes) < extent + && std::int64_t(j) >= cursor.skipped_ffts && j < active; + if (j < std::size_t(control_bits) && mask[j]) { + any = true; + } + } + if (!any) { + std::fill(mask.begin(), mask.end(), 0); + } + if (std::size_t(selected + 1) < state.anchors.size()) { + for (std::size_t j = 0; j < width; ++j) { + if (fact.ordinary_samples[j] + >= state.anchors[selected + 1]) { + mask[j] = 1; + } + } + } + } + for (std::size_t j = 0; j < width; ++j) { + if (std::int64_t(j) < cursor.skipped_ffts || j >= active) { + mask[j] = 0; + } + if (valid && j < active && !mask[j]) { + excluded.push_back(narrow(Wide(fact.first_fft) + j)); + } + } + out.allocations.push_back(selected); + out.cursors.push_back(cursor); + out.valid.push_back(valid); + out.events.push_back(event); + out.eligible.push_back(std::move(mask)); + } + std::sort(excluded.begin(), excluded.end()); + excluded.erase( + std::unique(excluded.begin(), excluded.end()), excluded.end()); + for (auto i : excluded) { + auto stop = narrow(Wide(i) + 1); + if (!out.excluded.empty() && out.excluded.back().stop == i) { + out.excluded.back().stop = stop; + } else { + out.excluded.push_back({i, stop}); + } + } + return out; +} +WindowBounds window_bounds( + std::int64_t output, + std::int64_t head, + double delay, + std::int64_t rate, + std::int64_t extent, + std::int64_t decoded, + std::int64_t fft) { + if (rate <= 0 || extent < 0 || decoded < 0 || fft <= 0 + || !std::isfinite(delay)) { + throw RadioError("invalid returned-window clock geometry"); + } + const double samples = delay * double(rate); + if (!std::isfinite(samples) || std::abs(samples) >= std::ldexp(1., 53)) { + throw RadioError("window delay exceeds exact sample-phase range"); + } + const auto whole = static_cast(std::floor(samples)); + Wide nearest = Wide(output) - head - whole; + bool integral = samples == double(whole); + const double midpoint = double(whole) + .5; + if (nearest < 0 && (integral || samples < midpoint)) { + ++nearest; + } + if (nearest > 0 && !integral && samples > midpoint) { + --nearest; + } + auto value = narrow(nearest); + auto first = nearest <= -Wide(fft) ? fft + : nearest < 0 ? narrow(-nearest) + : 0; + auto stop = nearest >= decoded ? 0 + : nearest > Wide(decoded) - fft ? narrow(Wide(decoded) - nearest) + : fft; + stop = std::max(stop, first); + return {value, first, stop, value<-1, nearest> Wide(extent) - fft}; +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/src/requests.cpp b/src/cuphoton/xbeam/native/src/requests.cpp new file mode 100644 index 00000000..7e8efaa0 --- /dev/null +++ b/src/cuphoton/xbeam/native/src/requests.cpp @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/radio.hpp" +#include +#include +namespace xbeam { +namespace { +using Wide = __int128; +Wide mod(Wide x, Wide y) { + auto r = x % y; + return r < 0 ? r + y : r; +} +std::int64_t narrow(Wide x) { + if (x < std::numeric_limits::min() + || x > std::numeric_limits::max()) { + throw RadioError("buffered request coordinate exceeds int64"); + } + return static_cast(x); +} +Wide round_sample(double x) { + x = std::trunc(x + 0.5); + if (!std::isfinite(x) || x < -std::ldexp(1.0, 126) + || x >= std::ldexp(1.0, 126)) { + throw RadioError("buffered delay drift exceeds supported coordinates"); + } + return narrow(static_cast(x)); +} +} // namespace +RequestCursor request_cursor( + std::int64_t local, + double drift_seconds, + const RequestGeometry& g, + double rate) { + if (!std::isfinite(drift_seconds)) { + throw RadioError("delay drift must be finite"); + } + if (!std::isfinite(rate) || rate <= 0) { + throw RadioError("sample rate must be finite and positive"); + } + if (g.ratio_numerator <= 0 || g.ratio_denominator <= 0 || g.fft_samples <= 0 + || g.ffts_per_request <= 0 || g.alignment_bytes <= 0 + || g.word_bytes <= 0) { + throw RadioError("buffered request geometry must be positive"); + } + Wide num = g.ratio_numerator, den = g.ratio_denominator; + if ((Wide(g.fft_samples) * den) % num || g.alignment_bytes % g.word_bytes + || (Wide(g.alignment_bytes) * num) % den) { + throw RadioError( + "buffered geometry must cover complete bytes, words and samples"); + } + const auto payload = narrow(Wide(g.fft_samples) * den / num); + Wide origin = Wide(local) * den / num; + origin -= mod(origin, g.word_bytes); + Wide skipped = 0; + if (origin < 0) { + skipped = (-origin + payload - 1) / payload; + if (skipped >= g.ffts_per_request) { + return {}; + } + origin += skipped * payload; + } + Wide drift = 0; + if (skipped) { + drift = round_sample( + static_cast(skipped) * drift_seconds * rate + / g.ffts_per_request); + const auto correction = drift * den / num; + if (origin - correction < 0) { + ++skipped; + origin += payload; + if (skipped == g.ffts_per_request) { + return {}; + } + drift = round_sample( + static_cast(skipped) * drift_seconds * rate + / g.ffts_per_request); + origin -= drift * den / num; + } + } + origin -= mod(origin, g.word_bytes); + origin -= mod(origin, g.alignment_bytes); + const auto origin64 = narrow(origin); + const Wide product = Wide(origin64) * num; + Wide presented = product / den; + if (product < 0 && product % den) { + --presented; + } + const Wide delta = + Wide(local) + skipped * g.fft_samples - drift - presented; + return {true, origin64, narrow(skipped), narrow(delta)}; +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/src/swin.cpp b/src/cuphoton/xbeam/native/src/swin.cpp new file mode 100644 index 00000000..7cf15e7a --- /dev/null +++ b/src/cuphoton/xbeam/native/src/swin.cpp @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/radio.hpp" +#include +#include +#include +#include +#include +#include +namespace xbeam { +namespace { +template +void append(std::string& output, const T value) { + static_assert(std::is_trivially_copyable::value, "scalar required"); + unsigned char bytes[sizeof(T)]; + std::memcpy(bytes, &value, sizeof(T)); + const std::uint16_t marker = 1; + const bool little = *reinterpret_cast(&marker) == 1; + for (std::size_t i = 0; i < sizeof(T); ++i) { + output.push_back( + static_cast(bytes[little ? i : sizeof(T) - 1 - i])); + } +} +} // namespace +std::string encode_swin( + const SwinMetadata& m, const float* samples, std::size_t channels) { + static_assert( + sizeof(float) == 4 && sizeof(double) == 8, "IEEE sizes required"); + static_assert( + std::numeric_limits::is_iec559 + && std::numeric_limits::is_iec559, + "IEEE floats required"); + const int a = m.baseline / 256, b = m.baseline % 256; + if (a < 1 || a > 255 || b < 1 || b > 255) { + throw RadioError("invalid SWIN baseline"); + } + if (static_cast(m.polarization_a) > 127 + || static_cast(m.polarization_b) > 127) { + throw RadioError("SWIN polarization must be ASCII"); + } + if (channels > (std::numeric_limits::max() - 74) / 8) { + throw RadioError("SWIN channel count overflow"); + } + if (channels && !samples) { + throw RadioError("missing SWIN spectrum"); + } + std::string out; + out.reserve(74 + 8 * channels); + append(out, std::uint32_t(0xff00ff00)); + append(out, std::uint32_t(1)); + append(out, m.baseline); + append(out, m.mjd); + append(out, m.seconds); + append(out, m.configuration_id); + append(out, m.source_id); + append(out, m.frequency_id); + out.push_back(m.polarization_a); + out.push_back(m.polarization_b); + append(out, m.pulsar_bin); + append(out, m.weight); + append(out, m.u); + append(out, m.v); + append(out, m.w); + for (std::size_t i = 0; i < 2 * channels; ++i) { + append(out, samples[i]); + } + return out; +} +void write_swin_file(const std::string& path, const std::string& records) { + if (path.find('\0') != std::string::npos) { + throw RadioError("path contains NUL"); + } + int fd = + ::open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666); + if (fd < 0) { + throw RadioError( + "cannot create SWIN output: " + std::string(std::strerror(errno))); + } + std::size_t written = 0; + while (written < records.size()) { + const auto n = + ::write(fd, records.data() + written, records.size() - written); + if (n < 0 && errno == EINTR) { + continue; + } + if (n <= 0) { + const int error = errno; + ::close(fd); + throw RadioError( + "SWIN write failed: " + std::string(std::strerror(error))); + } + written += static_cast(n); + } + if (::close(fd)) { + throw RadioError("SWIN close failed"); + } +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/src/timeline.cpp b/src/cuphoton/xbeam/native/src/timeline.cpp new file mode 100644 index 00000000..b1532580 --- /dev/null +++ b/src/cuphoton/xbeam/native/src/timeline.cpp @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/timeline.hpp" +#include "xbeam/radio.hpp" +#include +#include +namespace xbeam { +namespace { +using Wide = __int128; +std::int64_t checked(Wide x) { + if (x < INT64_MIN || x > INT64_MAX) { + throw RadioError("timeline coordinate exceeds int64"); + } + return static_cast(x); +} +std::int64_t ceil_div(std::int64_t x, std::int64_t y) { + return x / y + (x % y > 0); +} +Wide floor_div(Wide x, Wide y) { + return x / y - (x % y < 0); +} +void bounds(std::int64_t first, std::int64_t count, std::int64_t samples) { + if (first < 0 || count <= 0 || samples <= 0) { + throw RadioError("invalid timeline sample request"); + } +} +} // namespace +PhysicalTimeline::PhysicalTimeline( + const std::vector& input, std::int64_t allocation_frames) + : runs(input), + capacity(allocation_frames) { + if (capacity <= 0 || runs.empty() || runs.front().first != 0) { + throw RadioError( + "timeline requires positive allocation and zero-based runs"); + } + for (std::size_t i = 0; i < runs.size(); ++i) { + const auto& r = runs[i]; + if (r.first < 0 || r.timestamp < 0 || r.stop <= r.first + || (i && r.first < runs[i - 1].stop)) { + throw RadioError("invalid physical timeline runs"); + } + auto stop = checked(Wide(r.timestamp) + r.stop - r.first); + if (i + && Wide(r.timestamp) - timestamp_stops.back() + - (Wide(r.first) - runs[i - 1].stop) + < 0) { + throw RadioError("backwards timestamp residual"); + } + physical_stops.push_back(r.stop); + timestamp_stops.push_back(stop); + } + const auto allocation_count = ceil_div(runs.back().stop, capacity); + // Both vectors together have a bounded 256 MiB admission budget. + if (allocation_count > 16777216) { + throw RadioError("timeline allocation table exceeds memory budget"); + } + allocation_starts_.reserve(allocation_count); + anchors_.reserve(allocation_count); + for (std::int64_t i = 0; i < allocation_count; ++i) { + auto first = checked(Wide(i) * capacity); + auto pos = std::upper_bound( + physical_stops.begin(), physical_stops.end(), first) + - physical_stops.begin(); + if (std::size_t(pos) >= runs.size()) { + pos = runs.size() - 1; + } + allocation_starts_.push_back(first); + anchors_.push_back( + checked(Wide(runs[pos].timestamp) + first - runs[pos].first)); + } +} +std::vector PhysicalTimeline::uncovered_heads() const { + std::vector out; + for (std::size_t i = 1; i < runs.size(); ++i) { + if (runs[i].first > runs[i - 1].stop) { + auto first = ceil_div(runs[i - 1].stop, capacity), + stop = ceil_div(runs[i].first, capacity); + if (first >= stop) { + continue; + } + if (!out.empty() && first <= out.back().stop) { + out.back().stop = std::max(stop, out.back().stop); + } else { + out.push_back({first, stop}); + } + } + } + return out; +} +std::vector PhysicalTimeline::deferred_gaps() const { + std::vector out; + for (std::size_t i = 1; i < anchors_.size(); ++i) { + auto expected = checked( + Wide(anchors_[i - 1]) + allocation_starts_[i] + - allocation_starts_[i - 1]); + if (anchors_[i] > expected) { + out.push_back( + {static_cast(i), + allocation_starts_[i], + expected, + anchors_[i]}); + } + } + return out; +} +std::vector PhysicalTimeline::source_spans( + std::int64_t first, std::int64_t count, std::int64_t samples) const { + bounds(first, count, samples); + Wide end = Wide(first) + count; + std::vector out; + auto pos = + std::upper_bound( + timestamp_stops.begin(), timestamp_stops.end(), first / samples) + - timestamp_stops.begin(); + for (std::size_t i = pos; i < runs.size(); ++i) { + Wide a = Wide(runs[i].timestamp) * samples, + b = Wide(timestamp_stops[i]) * samples; + if (a >= end) { + break; + } + Wide begin = std::max(Wide(first), a), stop = std::min(end, b); + if (begin < stop) { + out.push_back( + {checked(begin - first), + checked(stop - first), + checked(Wide(runs[i].first) * samples + begin - a)}); + } + } + return out; +} +std::vector PhysicalTimeline::latched_spans( + std::int64_t allocation, + std::int64_t first, + std::int64_t count, + std::int64_t samples) const { + bounds(first, count, samples); + if (allocation < 0 || std::size_t(allocation) >= anchors_.size()) { + throw RadioError("allocation exceeds timeline"); + } + const Wide physical = Wide(allocation_starts_[allocation]) * samples + first + - Wide(anchors_[allocation]) * samples; + const Wide end = physical + count; + std::vector out; + auto origin = floor_div(physical, samples); + auto pos = std::upper_bound( + physical_stops.begin(), + physical_stops.end(), + origin, + [](Wide x, std::int64_t y) { return x < y; }) + - physical_stops.begin(); + for (std::size_t i = pos; i < runs.size(); ++i) { + Wide a = Wide(runs[i].first) * samples, + b = Wide(runs[i].stop) * samples; + if (a >= end) { + break; + } + Wide begin = std::max(physical, a), stop = std::min(end, b); + if (begin >= stop) { + continue; + } + SampleSpan span{checked(begin - physical), + checked(stop - physical), + checked(begin)}; + if (!out.empty() && out.back().request_stop == span.request_first + && Wide(out.back().physical_first) + out.back().request_stop + - out.back().request_first + == span.physical_first) { + out.back().request_stop = span.request_stop; + } else { + out.push_back(span); + } + } + return out; +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/src/vdif.cpp b/src/cuphoton/xbeam/native/src/vdif.cpp new file mode 100644 index 00000000..5f62b811 --- /dev/null +++ b/src/cuphoton/xbeam/native/src/vdif.cpp @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/radio.hpp" +#include +#include +#include +#include +#include +#include +namespace xbeam { +namespace { +std::uint32_t word(const unsigned char* p) { + return std::uint32_t(p[0]) | (std::uint32_t(p[1]) << 8) + | (std::uint32_t(p[2]) << 16) | (std::uint32_t(p[3]) << 24); +} +class File { +public: + explicit File(const std::string& path) + : fd(::open(path.c_str(), O_RDONLY | O_CLOEXEC)) { + if (fd < 0) { + throw RadioError( + "cannot open VDIF file: " + std::string(std::strerror(errno))); + } + } + ~File() { + ::close(fd); + } + File(const File&) = delete; + File& operator=(const File&) = delete; + int fd; +}; +} // namespace +VdifHeader parse_vdif_header( + const unsigned char* p, + std::size_t size, + bool strict, + bool require_two_bit) { + if (size < 16) { + throw RadioError("truncated VDIF header (requires at least 16 bytes)"); + } + const auto w0 = word(p), w1 = word(p + 4), w2 = word(p + 8), + w3 = word(p + 12); + if ((w0 >> 30) & 1) { + throw RadioError("legacy 16-byte VDIF headers are unsupported"); + } + if (size < 32) { + throw RadioError("truncated standard VDIF header (requires 32 bytes)"); + } + VdifHeader h{}; + h.invalid = w0 >> 31; + h.seconds_from_epoch = w0 & 0x3fffffff; + h.reference_epoch = (w1 >> 24) & 63; + h.frame_number = w1 & 0xffffff; + h.version = w2 >> 29; + h.log2_channels = (w2 >> 24) & 31; + h.frame_length_bytes = (w2 & 0xffffff) * 8; + h.complex_data = w3 >> 31; + h.bits_per_sample = ((w3 >> 26) & 31) + 1; + h.thread_id = (w3 >> 16) & 1023; + h.station_id = w3 & 65535; + for (unsigned i = 0; i < 4; ++i) { + h.extended_words[i] = word(p + 16 + 4 * i); + } + h.extended_data_version = h.extended_words[0] >> 24; + if (strict && w1 >> 30) { + throw RadioError("VDIF word 1 reserved bits must be zero"); + } + if (h.version) { + throw RadioError("unsupported VDIF version"); + } + if (h.frame_length_bytes < 32) { + throw RadioError("VDIF frame is smaller than its header"); + } + if (require_two_bit && h.bits_per_sample != 2) { + throw RadioError("unsupported VDIF quantization (expected 2 bits)"); + } + const std::uint64_t sample_bits = (std::uint64_t(1) << h.log2_channels) + * (h.complex_data ? 2 : 1) * h.bits_per_sample; + const std::uint64_t payload_bits = + std::uint64_t(h.frame_length_bytes - 32) * 8; + if (payload_bits % sample_bits) { + throw RadioError("VDIF payload has incomplete samples"); + } + h.samples_per_frame = payload_bits / sample_bits; + return h; +} +RecordingSummary inspect_vdif(const std::string& path, bool require_two_bit) { + File file(path); + struct stat before{}, after{}; + if (::fstat(file.fd, &before)) { + throw RadioError("cannot stat VDIF file"); + } + if (!S_ISREG(before.st_mode)) { + throw RadioError("VDIF input must be a regular file"); + } + RecordingSummary out; + out.source_bytes = before.st_size; + std::uint64_t offset = 0; + while (offset < out.source_bytes) { + unsigned char buffer[32]; + std::size_t n = 0; + while (n < 32) { + auto got = ::pread( + file.fd, buffer + n, 32 - n, static_cast(offset + n)); + if (got < 0 && errno == EINTR) { + continue; + } + if (got < 0) { + throw RadioError("VDIF header read failed"); + } + if (!got) { + throw RadioError( + "truncated VDIF header at byte " + std::to_string(offset)); + } + n += static_cast(got); + } + const auto h = parse_vdif_header(buffer, 32, true, require_two_bit); + if (h.frame_length_bytes > out.source_bytes - offset) { + throw RadioError( + "truncated VDIF frame at byte " + std::to_string(offset)); + } + if (!out.frames) { + out.frame_bytes = h.frame_length_bytes; + } else if (out.frame_bytes != h.frame_length_bytes) { + out.frame_bytes = 0; + } + ++out.frames; + out.invalid_frames += h.invalid; + ++out.thread_frames[h.thread_id]; + offset += h.frame_length_bytes; + } + if (::fstat(file.fd, &after) || before.st_size != after.st_size + || before.st_mtim.tv_sec != after.st_mtim.tv_sec + || before.st_mtim.tv_nsec != after.st_mtim.tv_nsec + || before.st_ctim.tv_sec != after.st_ctim.tv_sec + || before.st_ctim.tv_nsec != after.st_ctim.tv_nsec) { + throw RadioError("VDIF source changed during inspection"); + } + return out; +} +std::string inspect_vdif_json(const std::string& path) { + const auto s = inspect_vdif(path, false); + std::ostringstream out; + out << "{\"frames\":" << s.frames + << ",\"invalid_frames\":" << s.invalid_frames + << ",\"source_bytes\":" << s.source_bytes + << ",\"frame_bytes\":" << s.frame_bytes << ",\"threads\":["; + bool first = true; + for (std::size_t i = 0; i < s.thread_frames.size(); ++i) { + if (s.thread_frames[i]) { + if (!first) { + out << ','; + } + first = false; + out << "{\"thread_id\":" << i + << ",\"frames\":" << s.thread_frames[i] << '}'; + } + } + out << "]}"; + return out.str(); +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/src/window.cpp b/src/cuphoton/xbeam/native/src/window.cpp new file mode 100644 index 00000000..c819711f --- /dev/null +++ b/src/cuphoton/xbeam/native/src/window.cpp @@ -0,0 +1,518 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "xbeam/window.hpp" +#include +#include +#include +namespace xbeam { +namespace { +using I = __int128; +std::int64_t checked(I v) { + if (v < INT64_MIN || v > INT64_MAX) { + throw RadioError("returned window coordinate exceeds int64"); + } + return static_cast(v); +} +std::int64_t positive(I v) { + if (v <= 0) { + throw RadioError("window extent must be positive"); + } + return checked(v); +} +I floor_div(I a, I b) { + I q = a / b; + return q - (a % b < 0); +} +} // namespace +WindowGeometry size_windows( + std::int64_t spf, + std::int64_t fft, + std::int64_t rate, + std::int64_t num, + std::int64_t den, + std::int64_t factor, + std::int64_t segments, + std::int64_t buffered, + double allowance) { + if (spf <= 0 || fft <= 0 || rate <= 0 || num <= 0 || den <= 0 || factor <= 0 + || segments <= 0 || buffered <= 0 || fft % spf || factor % segments + || !std::isfinite(allowance) || allowance < 0) { + throw RadioError("invalid framed buffer geometry"); + } + const I product = I(num) * rate, divisor = I(den) * fft; + if (product % divisor) { + throw RadioError("request duration is not FFT aligned"); + } + auto width = positive(product / divisor), frames = fft / spf; + I nominal = I(width) * frames; + if (nominal >= (I(1) << 53)) { + throw RadioError("nominal frames exceed exact binary64 range"); + } + double increment = double(nominal) * allowance; + double expanded = double(nominal) + increment; + if (!std::isfinite(expanded) || expanded >= std::ldexp(1., 53) - 2.) { + throw RadioError("expanded frames exceed exact binary64 range"); + } + auto request = checked(I(static_cast(expanded)) + 2); + return {frames, + width, + buffered, + segments, + positive(I(factor / segments) * request), + request, + factor / segments}; +} +NormalExtent normal_extent( + std::int64_t frame, + std::int64_t payload, + std::int64_t requests, + std::int64_t ratio, + std::int64_t num, + std::int64_t den) { + if (frame <= 0 || payload <= 0 || payload > frame || requests <= 0 + || ratio <= 0 || num <= 0 || den <= 0) { + throw RadioError("invalid normal returned extent"); + } + const auto allocation_frames = positive(I(requests) * ratio); + const auto allocation_bytes = positive(I(allocation_frames) * frame); + // Both factors are now bounded by the validated allocation extent. + I spf = I(payload) * num, admission = I(requests) * frame * num; + if (spf % den || admission % den) { + throw RadioError("normal return requires integral samples"); + } + return {frame, + payload, + requests, + ratio, + num, + den, + positive(spf / den), + positive(admission / den), + positive(I(requests) * (spf / den)), + allocation_bytes}; +} +std::int64_t leading_head( + std::int64_t output, + double endpoint, + std::int64_t rate, + std::int64_t spf, + std::int64_t quantum) { + if (rate <= 0 || spf <= 0 || quantum <= 0 || !std::isfinite(endpoint)) { + throw RadioError("invalid leading head clock"); + } + double units = std::trunc(endpoint * (1000000000. / double(quantum))); + if (!std::isfinite(units) || units < -std::ldexp(1., 63) + || units >= std::ldexp(1., 63)) { + throw RadioError("endpoint nanoseconds exceed int64"); + } + I ns = checked(I(static_cast(units)) * quantum); + // ns * rate can span up to 126 bits; the subsequent subtraction stays + // within signed 128 for validated int64 operands and billion-scale term. + I position = I(output) * 1000000000 - ns * rate; + return checked(floor_div(position, I(1000000000) * spf) * spf); +} +} // namespace xbeam +namespace xbeam { +std::vector returned_windows( + const PhysicalTimeline& timeline, + const RawRequestState& state, + const JobRequestInputs& inputs, + const RequestGeometry& g, + const NormalExtent& extent, + std::int64_t control, + std::int64_t quantum, + std::int64_t source_stop) { + if (inputs.sample_rate != std::trunc(inputs.sample_rate) + || inputs.sample_rate <= 0 || inputs.sample_rate >= std::ldexp(1., 63) + || control <= 0 || quantum <= 0 + || state.timestamp_origin != inputs.timestamp_origin + || g.ratio_numerator != extent.ratio_num + || g.ratio_denominator != extent.ratio_den + || state.samples_per_frame != extent.samples_per_frame + || I(extent.request_frames) * extent.read_buffer_ratio + != state.allocation_frames) { + throw RadioError("returned contract and raw geometry differ"); + } + const auto rate = static_cast(inputs.sample_rate); + const auto& runs = timeline.physical_runs(); + if (runs.empty() + || source_stop + < checked(I(runs.back().stop) * state.samples_per_frame)) { + throw RadioError("known source end excludes physical history"); + } + const auto batch = compile_discards( + state, + inputs.requests, + g, + inputs.sample_rate, + extent.frame_bytes, + control); + std::vector result; + if (inputs.stop_fft - inputs.first_fft > 16777216) { + throw RadioError("returned scope exceeds memory budget"); + } + result.reserve(inputs.stop_fft - inputs.first_fft); + const auto uncovered = timeline.uncovered_heads(); + auto in_hole = [&](std::int64_t owner) { + auto it = std::upper_bound( + uncovered.begin(), + uncovered.end(), + owner, + [](auto v, const auto& r) { return v < r.stop; }); + return it != uncovered.end() && owner >= it->first; + }; + auto next_run = [&](std::int64_t frame) { + return std::upper_bound( + runs.begin(), runs.end(), frame, [](auto v, const auto& r) { + return v < r.stop; + }); + }; + const auto spf = state.samples_per_frame, + capacity = state.allocation_frames; + const auto header_prefix = checked( + (I(capacity) * (extent.frame_bytes - extent.payload_bytes) + * extent.ratio_num + + extent.ratio_den - 1) + / extent.ratio_den); + bool seen_normal = false; + for (std::size_t row = 0; row < inputs.requests.size(); ++row) { + const auto& fact = inputs.requests[row]; + const auto active = fact.active_count + ? fact.active_count + : std::int64_t(fact.ordinary_samples.size()); + const auto width = fact.ordinary_samples.size(); + if (inputs.output_samples.at(row).size() != width + || inputs.centre_delays.at(row).size() != width) { + throw RadioError( + "returned clock matrices have inconsistent shapes"); + } + const bool initial = fact.desired_sample < state.anchors.front(); + if (initial && seen_normal) { + throw RadioError("initial requests must form a prefix"); + } + seen_normal |= !initial; + std::int64_t selected = batch.allocations[row], allocation = selected, + cursor = 0, head = 0, original_head = 0; + std::vector eligible(width, 0); + bool event = false, transfer = false; + if (initial) { + if (state.anchors.front() != 0 + || timeline.allocation_starts().front() != 0) { + throw RadioError("initial source must start at job origin"); + } + std::int64_t end = 0; + for (const auto& run : runs) { + if (run.first != end + || I(run.timestamp) - state.timestamp_origin != run.first) { + break; + } + end = run.stop; + if (end >= capacity) { + break; + } + } + if (end < capacity) { + throw RadioError("initial allocation clock is not contiguous"); + } + auto c = request_cursor( + fact.desired_sample, + fact.endpoint_drift, + g, + inputs.sample_rate); + for (std::size_t j = 0; j < width; ++j) { + eligible[j] = c.valid && std::int64_t(j) < active + && std::int64_t(j) >= c.skipped_ffts; + } + allocation = 0; + } else { + if (!batch.valid[row]) { + throw RadioError("returned aligned head is unresolved"); + } + event = batch.events[row]; + cursor = batch.cursors[row].origin_bytes; + if (cursor % g.ratio_denominator) { + throw RadioError("aligned cursor does not name a sample"); + } + original_head = head = checked( + I(state.anchors[selected]) + + I(cursor / g.ratio_denominator) * g.ratio_numerator); + bool control_present = false; + for (std::size_t j = 0; j < width; ++j) { + bool valid = std::int64_t(j) < active + && std::int64_t(j) >= batch.cursors[row].skipped_ffts; + if (event) { + valid &= I(cursor) + + I(j) * g.fft_samples * g.ratio_denominator + / g.ratio_numerator + < extent.allocation_bytes; + } + eligible[j] = valid; + if (j < std::size_t(control) && valid) { + control_present = true; + } + } + if (event && !control_present) { + std::fill(eligible.begin(), eligible.end(), 0); + } + transfer = event + && std::none_of( + eligible.begin(), eligible.end(), [](auto x) { return x; }); + if (transfer) { + ++allocation; + if (std::size_t(allocation) >= state.anchors.size()) { + throw RadioError("successor window lies outside history"); + } + cursor = 0; + head = state.anchors[allocation]; + auto c = request_cursor( + checked(I(fact.desired_sample) - head), + fact.endpoint_drift, + g, + inputs.sample_rate); + for (std::size_t j = 0; j < width; ++j) { + eligible[j] = c.valid && std::int64_t(j) < active + && std::int64_t(j) >= c.skipped_ffts; + } + } + } + const auto physical_head = initial + ? 0 + : checked( + I(timeline.allocation_starts().at(allocation)) * spf + + I(cursor / g.ratio_denominator) * g.ratio_numerator); + for (std::int64_t j = 0; j < active; ++j) { + auto fft = checked(I(fact.first_fft) + j); + if (fft < inputs.first_fft || fft >= inputs.stop_fft) { + continue; + } + ReturnedWindow window{fft, + inputs.output_samples[row][j], + head, + physical_head, + inputs.centre_delays[row][j], + extent.framed_admission_samples, + extent.decoded_samples, + bool(eligible[j]), + initial}; + if (!initial && !event && j == 0 && head == original_head) { + auto replacement = leading_head( + window.output, + inputs.left_delays.at(row), + rate, + spf, + quantum); + window.physical_head = + checked(I(physical_head) + replacement - head); + window.logical_head = replacement; + window.clamp_negative = true; + } + auto bounds = window_bounds( + window.output, + window.logical_head, + window.centre_delay, + rate, + window.framed_extent, + window.decoded_extent, + g.fft_samples); + auto physical = checked( + I(window.physical_head) + + std::max(bounds.nearest, 0)); + const bool unchanged = + !initial && window.logical_head == original_head; + if (!initial && event && unchanged && !bounds.before + && !bounds.after) { + auto framehead = timeline.allocation_starts().at(selected); + auto r = next_run(framehead); + bool covered = r != runs.end() && r->first <= framehead; + auto boundary = checked((I(framehead) + capacity) * spf); + bool withheld = (j == 0 || !covered) && physical >= boundary; + auto framed_samples = checked( + (I(extent.allocation_bytes) * g.ratio_numerator + + g.ratio_denominator - 1) + / g.ratio_denominator); + auto allowance = checked(I(framed_samples) - I(capacity) * spf); + if (!withheld && window.published + && physical >= checked(I(boundary) + allowance) + && std::size_t(selected + 1) < state.anchors.size()) { + withheld = I(state.anchors[selected + 1]) + > I(state.anchors[selected]) + I(capacity) * spf; + } + window.published &= !withheld; + } + // Initial prefix publication is a separate contract and does not + // inherit the normal-return allocation-head gate. + if (!initial) { + bool published = + window.published && !bounds.before && !bounds.after; + const auto end = checked(I(physical) + g.fft_samples); + auto r = next_run(physical / spf); + bool covered = r != runs.end() && I(r->first) * spf < end; + bool empty = published && !covered; + double units = std::trunc( + window.centre_delay * (1000000000. / double(quantum))); + auto ownbounds = window_bounds( + window.output, + window.logical_head, + units * (double(quantum) / 1000000000.), + rate, + window.framed_extent, + window.decoded_extent, + g.fft_samples); + auto ownership = checked( + I(window.physical_head) + + std::max(ownbounds.nearest, 0)); + auto owner = ownership / spf / capacity; + bool withheld = in_hole(owner); + if (withheld && published) { + auto hole_head = checked(I(owner) * capacity), + head_sample = checked(I(hole_head) * spf); + auto resume = next_run(hole_head); + if (resume == runs.end() || resume == runs.begin()) { + throw RadioError( + "uncovered allocation has no trustworthy successor"); + } + const auto& previous = *(resume - 1); + I residual = I(resume->timestamp) - previous.timestamp + - (I(resume->first) - previous.first); + auto decoded_stop = checked( + I(window.physical_head) + window.decoded_extent); + bool retained = residual == 0 + && !in_hole(window.physical_head / spf / capacity) + && window.physical_head < head_sample + && physical < checked(I(head_sample) + header_prefix) + && std::max(physical, checked(I(resume->first) * spf)) + < std::min(end, decoded_stop); + if (retained) { + withheld = false; + } + } + window.published &= !withheld && !empty; + } + if (window.physical_head < 0) { + throw RadioError("negative retained physical head"); + } + window.decoded_extent = std::min( + window.decoded_extent, + std::max( + checked(I(source_stop) - window.physical_head), 0)); + result.push_back(window); + } + } + if (result.size() != std::size_t(inputs.stop_fft - inputs.first_fft)) { + throw RadioError("returned requests do not cover selected scope"); + } + for (std::size_t k = 0; k < result.size(); ++k) { + if (result[k].fft != inputs.first_fft + std::int64_t(k)) { + throw RadioError("returned requests are not contiguous"); + } + } + return result; +} +} // namespace xbeam +namespace xbeam { +std::vector compile_replay_rows( + const std::vector& windows, + std::int64_t fft, + std::int64_t rate, + std::int64_t stop) { + if (fft <= 0 || rate <= 0 || stop < 0) { + throw RadioError("invalid replay source extent"); + } + std::vector rows; + rows.reserve(windows.size()); + for (const auto& w : windows) { + auto bounds = window_bounds( + w.output, + w.logical_head, + w.centre_delay, + rate, + w.framed_extent, + w.decoded_extent, + fft); + auto physical = checked( + I(w.physical_head) + std::max(bounds.nearest, 0)); + if (w.physical_head < 0 + || (I(w.physical_head) + w.decoded_extent > stop + && w.decoded_extent > 0)) { + throw RadioError("retained window exceeds declared source end"); + } + bool accepted = + w.published && !bounds.before && !bounds.after && physical < stop; + if (w.clamp_negative && bounds.nearest < 0) { + bounds = window_bounds( + w.logical_head, + w.logical_head, + 0, + rate, + w.framed_extent, + w.decoded_extent, + fft); + } + const auto logical = checked( + I(w.logical_head) + + std::max( + window_bounds( + w.output, + w.logical_head, + w.centre_delay, + rate, + w.framed_extent, + w.decoded_extent, + fft) + .nearest, + 0)); + double coarse = std::nearbyint(w.centre_delay * double(rate)); + if (!std::isfinite(coarse) || coarse < -std::ldexp(1., 63) + || coarse >= std::ldexp(1., 63)) { + throw RadioError("replay centre delay exceeds int64 samples"); + } + auto displacement = + checked(I(logical) - w.output + static_cast(coarse)); + rows.push_back( + {w.fft, + physical, + displacement, + accepted ? bounds.valid_first : 0, + accepted ? bounds.valid_stop : 0, + accepted}); + } + return rows; +} +} // namespace xbeam + +namespace xbeam { +std::int64_t reference_epoch_seconds(int epoch) { + if (epoch < 0 || epoch > 63) { + throw RadioError("VDIF reference epoch is outside its six-bit range"); + } + const int year = 2000 + epoch / 2; + std::int64_t days = 0; + auto leap = [](int y) { + return y % 4 == 0 && (y % 100 != 0 || y % 400 == 0); + }; + for (int y = 2000; y < year; ++y) { + days += 365 + leap(y); + } + if (epoch % 2) { + days += 181 + leap(year); + } + // UTC leap transitions are part of the shared VDIF time convention. + const int transitions[] = {12, 18, 25, 31, 34}; + int leaps = 0; + for (int t : transitions) { + leaps += epoch >= t; + } + return days * 86400 + leaps; +} +std::int64_t packed_frame_slot(std::uint64_t stamp, std::int64_t fps) { + if (stamp >= (std::uint64_t(1) << 60) || fps <= 0 || fps > (1 << 24)) { + throw RadioError("invalid packed VDIF timestamp"); + } + auto frame = stamp & 0xffffff; + if (frame >= std::uint64_t(fps)) { + throw RadioError("VDIF frame number exceeds rate"); + } + auto seconds = (stamp >> 24) & 0x3fffffff; + return checked( + (I(reference_epoch_seconds(stamp >> 54)) + seconds) * fps + frame); +} +} // namespace xbeam diff --git a/src/cuphoton/xbeam/native/third_party/nlohmann/LICENSE.MIT b/src/cuphoton/xbeam/native/third_party/nlohmann/LICENSE.MIT new file mode 100644 index 00000000..b5a10275 --- /dev/null +++ b/src/cuphoton/xbeam/native/third_party/nlohmann/LICENSE.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2013-2025 Niels Lohmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/cuphoton/xbeam/native/third_party/nlohmann/json.hpp b/src/cuphoton/xbeam/native/third_party/nlohmann/json.hpp new file mode 100644 index 00000000..82d69f7c --- /dev/null +++ b/src/cuphoton/xbeam/native/third_party/nlohmann/json.hpp @@ -0,0 +1,25526 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + +/****************************************************************************\ + * Note on documentation: The source files contain links to the online * + * documentation of the public API at https://json.nlohmann.me. This URL * + * contains the most recent documentation and should also be applicable to * + * previous versions; documentation for deprecated functions is not * + * removed, but marked deprecated. See "Generate documentation" section in * + * file docs/README.md. * +\****************************************************************************/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#include // all_of, find, for_each +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#ifndef JSON_NO_IO + #include // istream, ostream +#endif // JSON_NO_IO +#include // random_access_iterator_tag +#include // unique_ptr +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// This file contains all macro definitions affecting or depending on the ABI + +#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK + #if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH) + #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 12 || NLOHMANN_JSON_VERSION_PATCH != 0 + #warning "Already included a different version of the library!" + #endif + #endif +#endif + +#define NLOHMANN_JSON_VERSION_MAJOR 3 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_MINOR 12 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_PATCH 0 // NOLINT(modernize-macro-to-enum) + +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 0 +#endif + +#ifndef JSON_DIAGNOSTIC_POSITIONS + #define JSON_DIAGNOSTIC_POSITIONS 0 +#endif + +#ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON + #define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0 +#endif + +#if JSON_DIAGNOSTICS + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag +#else + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS +#endif + +#if JSON_DIAGNOSTIC_POSITIONS + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS _dp +#else + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS +#endif + +#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON + #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp +#else + #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION + #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 +#endif + +// Construct the namespace ABI tags component +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c) json_abi ## a ## b ## c +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c) \ + NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c) + +#define NLOHMANN_JSON_ABI_TAGS \ + NLOHMANN_JSON_ABI_TAGS_CONCAT( \ + NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \ + NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON, \ + NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS) + +// Construct the namespace version component +#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ + _v ## major ## _ ## minor ## _ ## patch +#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(major, minor, patch) \ + NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) + +#if NLOHMANN_JSON_NAMESPACE_NO_VERSION +#define NLOHMANN_JSON_NAMESPACE_VERSION +#else +#define NLOHMANN_JSON_NAMESPACE_VERSION \ + NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(NLOHMANN_JSON_VERSION_MAJOR, \ + NLOHMANN_JSON_VERSION_MINOR, \ + NLOHMANN_JSON_VERSION_PATCH) +#endif + +// Combine namespace components +#define NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) a ## b +#define NLOHMANN_JSON_NAMESPACE_CONCAT(a, b) \ + NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) + +#ifndef NLOHMANN_JSON_NAMESPACE +#define NLOHMANN_JSON_NAMESPACE \ + nlohmann::NLOHMANN_JSON_NAMESPACE_CONCAT( \ + NLOHMANN_JSON_ABI_TAGS, \ + NLOHMANN_JSON_NAMESPACE_VERSION) +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_BEGIN +#define NLOHMANN_JSON_NAMESPACE_BEGIN \ + namespace nlohmann \ + { \ + inline namespace NLOHMANN_JSON_NAMESPACE_CONCAT( \ + NLOHMANN_JSON_ABI_TAGS, \ + NLOHMANN_JSON_NAMESPACE_VERSION) \ + { +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_END +#define NLOHMANN_JSON_NAMESPACE_END \ + } /* namespace (inline namespace) NOLINT(readability/namespace) */ \ + } // namespace nlohmann +#endif + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // transform +#include // array +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#ifdef JSON_HAS_CPP_17 + #include // optional +#endif +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // nullptr_t +#include // exception +#if JSON_DIAGNOSTICS + #include // accumulate +#endif +#include // runtime_error +#include // to_string +#include // vector + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // uint8_t +#include // string + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // declval, pair +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +// https://en.cppreference.com/w/cpp/experimental/is_detected +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template class Op, class... Args> +using is_detected = typename detector::value_t; + +template class Op, class... Args> +struct is_detected_lazy : is_detected { }; + +template class Op, class... Args> +using detected_t = typename detector::type; + +template class Op, class... Args> +using detected_or = detector; + +template class Op, class... Args> +using detected_or_t = typename detected_or::type; + +template class Op, class... Args> +using is_detected_exact = std::is_same>; + +template class Op, class... Args> +using is_detected_convertible = + std::is_convertible, To>; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + + +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-FileCopyrightText: 2016 - 2021 Evan Nemerson +// SPDX-License-Identifier: MIT + +/* Hedley - https://nemequ.github.io/hedley + * Created by Evan Nemerson + */ + +#if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 15) +#if defined(JSON_HEDLEY_VERSION) + #undef JSON_HEDLEY_VERSION +#endif +#define JSON_HEDLEY_VERSION 15 + +#if defined(JSON_HEDLEY_STRINGIFY_EX) + #undef JSON_HEDLEY_STRINGIFY_EX +#endif +#define JSON_HEDLEY_STRINGIFY_EX(x) #x + +#if defined(JSON_HEDLEY_STRINGIFY) + #undef JSON_HEDLEY_STRINGIFY +#endif +#define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x) + +#if defined(JSON_HEDLEY_CONCAT_EX) + #undef JSON_HEDLEY_CONCAT_EX +#endif +#define JSON_HEDLEY_CONCAT_EX(a,b) a##b + +#if defined(JSON_HEDLEY_CONCAT) + #undef JSON_HEDLEY_CONCAT +#endif +#define JSON_HEDLEY_CONCAT(a,b) JSON_HEDLEY_CONCAT_EX(a,b) + +#if defined(JSON_HEDLEY_CONCAT3_EX) + #undef JSON_HEDLEY_CONCAT3_EX +#endif +#define JSON_HEDLEY_CONCAT3_EX(a,b,c) a##b##c + +#if defined(JSON_HEDLEY_CONCAT3) + #undef JSON_HEDLEY_CONCAT3 +#endif +#define JSON_HEDLEY_CONCAT3(a,b,c) JSON_HEDLEY_CONCAT3_EX(a,b,c) + +#if defined(JSON_HEDLEY_VERSION_ENCODE) + #undef JSON_HEDLEY_VERSION_ENCODE +#endif +#define JSON_HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision)) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR) + #undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MINOR) + #undef JSON_HEDLEY_VERSION_DECODE_MINOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_REVISION) + #undef JSON_HEDLEY_VERSION_DECODE_REVISION +#endif +#define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000) + +#if defined(JSON_HEDLEY_GNUC_VERSION) + #undef JSON_HEDLEY_GNUC_VERSION +#endif +#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#elif defined(__GNUC__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) +#endif + +#if defined(JSON_HEDLEY_GNUC_VERSION_CHECK) + #undef JSON_HEDLEY_GNUC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GNUC_VERSION) + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION) + #undef JSON_HEDLEY_MSVC_VERSION +#endif +#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100) +#elif defined(_MSC_FULL_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) +#elif defined(_MSC_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION_CHECK) + #undef JSON_HEDLEY_MSVC_VERSION_CHECK +#endif +#if !defined(JSON_HEDLEY_MSVC_VERSION) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0) +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) +#elif defined(_MSC_VER) && (_MSC_VER >= 1200) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) +#else + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor))) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION) + #undef JSON_HEDLEY_INTEL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) +#elif defined(__INTEL_COMPILER) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_VERSION) + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #undef JSON_HEDLEY_INTEL_CL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && defined(__ICL) + #define JSON_HEDLEY_INTEL_CL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER, __INTEL_COMPILER_UPDATE, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_CL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION) + #undef JSON_HEDLEY_PGI_VERSION +#endif +#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) + #define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION_CHECK) + #undef JSON_HEDLEY_PGI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PGI_VERSION) + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #undef JSON_HEDLEY_SUNPRO_VERSION +#endif +#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) +#elif defined(__SUNPRO_C) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf) +#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10) +#elif defined(__SUNPRO_CC) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK) + #undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#endif +#if defined(__EMSCRIPTEN__) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION) + #undef JSON_HEDLEY_ARM_VERSION +#endif +#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100) +#elif defined(__CC_ARM) && defined(__ARMCC_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION_CHECK) + #undef JSON_HEDLEY_ARM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_ARM_VERSION) + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION) + #undef JSON_HEDLEY_IBM_VERSION +#endif +#if defined(__ibmxl__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) +#elif defined(__xlC__) && defined(__xlC_ver__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) +#elif defined(__xlC__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION_CHECK) + #undef JSON_HEDLEY_IBM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IBM_VERSION) + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_VERSION) + #undef JSON_HEDLEY_TI_VERSION +#endif +#if \ + defined(__TI_COMPILER_VERSION__) && \ + ( \ + defined(__TMS470__) || defined(__TI_ARM__) || \ + defined(__MSP430__) || \ + defined(__TMS320C2000__) \ + ) +#if (__TI_COMPILER_VERSION__ >= 16000000) + #define JSON_HEDLEY_TI_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif +#endif + +#if defined(JSON_HEDLEY_TI_VERSION_CHECK) + #undef JSON_HEDLEY_TI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_VERSION) + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #undef JSON_HEDLEY_TI_CL2000_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C2000__) + #define JSON_HEDLEY_TI_CL2000_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL2000_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #undef JSON_HEDLEY_TI_CL430_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__MSP430__) + #define JSON_HEDLEY_TI_CL430_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL430_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL430_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #undef JSON_HEDLEY_TI_ARMCL_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && (defined(__TMS470__) || defined(__TI_ARM__)) + #define JSON_HEDLEY_TI_ARMCL_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION_CHECK) + #undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_ARMCL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #undef JSON_HEDLEY_TI_CL6X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C6X__) + #define JSON_HEDLEY_TI_CL6X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL6X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #undef JSON_HEDLEY_TI_CL7X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__C7000__) + #define JSON_HEDLEY_TI_CL7X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL7X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #undef JSON_HEDLEY_TI_CLPRU_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__PRU__) + #define JSON_HEDLEY_TI_CLPRU_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CLPRU_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION) + #undef JSON_HEDLEY_CRAY_VERSION +#endif +#if defined(_CRAYC) + #if defined(_RELEASE_PATCHLEVEL) + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) + #else + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION_CHECK) + #undef JSON_HEDLEY_CRAY_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_CRAY_VERSION) + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION) + #undef JSON_HEDLEY_IAR_VERSION +#endif +#if defined(__IAR_SYSTEMS_ICC__) + #if __VER__ > 1000 + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) + #else + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(__VER__ / 100, __VER__ % 100, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION_CHECK) + #undef JSON_HEDLEY_IAR_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IAR_VERSION) + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION) + #undef JSON_HEDLEY_TINYC_VERSION +#endif +#if defined(__TINYC__) + #define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION_CHECK) + #undef JSON_HEDLEY_TINYC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION) + #undef JSON_HEDLEY_DMC_VERSION +#endif +#if defined(__DMC__) + #define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION_CHECK) + #undef JSON_HEDLEY_DMC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_DMC_VERSION) + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #undef JSON_HEDLEY_COMPCERT_VERSION +#endif +#if defined(__COMPCERT_VERSION__) + #define JSON_HEDLEY_COMPCERT_VERSION JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK) + #undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION) + #undef JSON_HEDLEY_PELLES_VERSION +#endif +#if defined(__POCC__) + #define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION_CHECK) + #undef JSON_HEDLEY_PELLES_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PELLES_VERSION) + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #undef JSON_HEDLEY_MCST_LCC_VERSION +#endif +#if defined(__LCC__) && defined(__LCC_MINOR__) + #define JSON_HEDLEY_MCST_LCC_VERSION JSON_HEDLEY_VERSION_ENCODE(__LCC__ / 100, __LCC__ % 100, __LCC_MINOR__) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION_CHECK) + #undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_MCST_LCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION) + #undef JSON_HEDLEY_GCC_VERSION +#endif +#if \ + defined(JSON_HEDLEY_GNUC_VERSION) && \ + !defined(__clang__) && \ + !defined(JSON_HEDLEY_INTEL_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_ARM_VERSION) && \ + !defined(JSON_HEDLEY_CRAY_VERSION) && \ + !defined(JSON_HEDLEY_TI_VERSION) && \ + !defined(JSON_HEDLEY_TI_ARMCL_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL430_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL2000_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL6X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL7X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CLPRU_VERSION) && \ + !defined(__COMPCERT__) && \ + !defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_ATTRIBUTE +#endif +#if \ + defined(__has_attribute) && \ + ( \ + (!defined(JSON_HEDLEY_IAR_VERSION) || JSON_HEDLEY_IAR_VERSION_CHECK(8,5,9)) \ + ) +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) +#else +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#endif +#if \ + defined(__has_cpp_attribute) && \ + defined(__cplusplus) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS +#endif +#if !defined(__cplusplus) || !defined(__has_cpp_attribute) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#elif \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \ + (!defined(JSON_HEDLEY_MSVC_VERSION) || JSON_HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(ns::attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_BUILTIN) + #undef JSON_HEDLEY_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) +#else + #define JSON_HEDLEY_HAS_BUILTIN(builtin) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN) + #undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_BUILTIN) + #undef JSON_HEDLEY_GCC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_FEATURE) + #undef JSON_HEDLEY_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature) +#else + #define JSON_HEDLEY_HAS_FEATURE(feature) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_FEATURE) + #undef JSON_HEDLEY_GNUC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_FEATURE) + #undef JSON_HEDLEY_GCC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_EXTENSION) + #undef JSON_HEDLEY_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) +#else + #define JSON_HEDLEY_HAS_EXTENSION(extension) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION) + #undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_EXTENSION) + #undef JSON_HEDLEY_GCC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_WARNING) + #undef JSON_HEDLEY_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning) +#else + #define JSON_HEDLEY_HAS_WARNING(warning) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_WARNING) + #undef JSON_HEDLEY_GNUC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_WARNING) + #undef JSON_HEDLEY_GCC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + defined(__clang__) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \ + (JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR)) + #define JSON_HEDLEY_PRAGMA(value) _Pragma(#value) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_PRAGMA(value) __pragma(value) +#else + #define JSON_HEDLEY_PRAGMA(value) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH) + #undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#endif +#if defined(JSON_HEDLEY_DIAGNOSTIC_POP) + #undef JSON_HEDLEY_DIAGNOSTIC_POP +#endif +#if defined(__clang__) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) + #define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) +#elif JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_PUSH + #define JSON_HEDLEY_DIAGNOSTIC_POP +#endif + +/* JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat") +# if JSON_HEDLEY_HAS_WARNING("-Wc++17-extensions") +# if JSON_HEDLEY_HAS_WARNING("-Wc++1z-extensions") +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + _Pragma("clang diagnostic ignored \"-Wc++1z-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# endif +#endif +#if !defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(x) x +#endif + +#if defined(JSON_HEDLEY_CONST_CAST) + #undef JSON_HEDLEY_CONST_CAST +#endif +#if defined(__cplusplus) +# define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) +#elif \ + JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_CONST_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_REINTERPRET_CAST) + #undef JSON_HEDLEY_REINTERPRET_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) +#else + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_STATIC_CAST) + #undef JSON_HEDLEY_STATIC_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) +#else + #define JSON_HEDLEY_STATIC_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_CPP_CAST) + #undef JSON_HEDLEY_CPP_CAST +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wold-style-cast") +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wold-style-cast\"") \ + ((T) (expr)) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# elif JSON_HEDLEY_IAR_VERSION_CHECK(8,3,0) +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("diag_suppress=Pe137") \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_CPP_CAST(T, expr) ((T) (expr)) +# endif +#else +# define JSON_HEDLEY_CPP_CAST(T, expr) (expr) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wdeprecated-declarations") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:1478 1786)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1216,1444,1445") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:161)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068)) +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(16,9,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 161") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-attributes") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("clang diagnostic ignored \"-Wunknown-attributes\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("warning(disable:1292)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:1292)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:5030)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097,1098") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("error_messages(off,attrskipunsup)") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1173") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress=Pe1097") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wcast-qual") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunused-function") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("clang diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("GCC diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(1,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION __pragma(warning(disable:4505)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("diag_suppress 3142") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif + +#if defined(JSON_HEDLEY_DEPRECATED) + #undef JSON_HEDLEY_DEPRECATED +#endif +#if defined(JSON_HEDLEY_DEPRECATED_FOR) + #undef JSON_HEDLEY_DEPRECATED_FOR +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " # since)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) +#elif \ + (JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__("Since " #since "; use " #replacement))) +#elif defined(__cplusplus) && (__cplusplus >= 201402L) + #define JSON_HEDLEY_DEPRECATED(since) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since)]]) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since "; use " #replacement)]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_PELLES_VERSION_CHECK(6,50,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated") + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") +#else + #define JSON_HEDLEY_DEPRECATED(since) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) +#endif + +#if defined(JSON_HEDLEY_UNAVAILABLE) + #undef JSON_HEDLEY_UNAVAILABLE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warning) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) +#else + #define JSON_HEDLEY_UNAVAILABLE(available_since) +#endif + +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT +#endif +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT_MSG) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) __attribute__((__warn_unused_result__)) +#elif (JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard(msg)]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) +#elif defined(_Check_return_) /* SAL */ + #define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_ + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) _Check_return_ +#else + #define JSON_HEDLEY_WARN_UNUSED_RESULT + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) +#endif + +#if defined(JSON_HEDLEY_SENTINEL) + #undef JSON_HEDLEY_SENTINEL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) +#else + #define JSON_HEDLEY_SENTINEL(position) +#endif + +#if defined(JSON_HEDLEY_NO_RETURN) + #undef JSON_HEDLEY_NO_RETURN +#endif +#if JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NO_RETURN __noreturn +#elif \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L + #define JSON_HEDLEY_NO_RETURN _Noreturn +#elif defined(__cplusplus) && (__cplusplus >= 201103L) + #define JSON_HEDLEY_NO_RETURN JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[noreturn]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,2,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_NO_RETURN _Pragma("does_not_return") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NO_RETURN __attribute((noreturn)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#else + #define JSON_HEDLEY_NO_RETURN +#endif + +#if defined(JSON_HEDLEY_NO_ESCAPE) + #undef JSON_HEDLEY_NO_ESCAPE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(noescape) + #define JSON_HEDLEY_NO_ESCAPE __attribute__((__noescape__)) +#else + #define JSON_HEDLEY_NO_ESCAPE +#endif + +#if defined(JSON_HEDLEY_UNREACHABLE) + #undef JSON_HEDLEY_UNREACHABLE +#endif +#if defined(JSON_HEDLEY_UNREACHABLE_RETURN) + #undef JSON_HEDLEY_UNREACHABLE_RETURN +#endif +#if defined(JSON_HEDLEY_ASSUME) + #undef JSON_HEDLEY_ASSUME +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_ASSUME(expr) __assume(expr) +#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume) + #define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr) +#elif \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #if defined(__cplusplus) + #define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr) + #else + #define JSON_HEDLEY_ASSUME(expr) _nassert(expr) + #endif +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(10,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable() +#elif defined(JSON_HEDLEY_ASSUME) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif +#if !defined(JSON_HEDLEY_ASSUME) + #if defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, ((expr) ? 1 : (JSON_HEDLEY_UNREACHABLE(), 1))) + #else + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, expr) + #endif +#endif +#if defined(JSON_HEDLEY_UNREACHABLE) + #if \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (JSON_HEDLEY_STATIC_CAST(void, JSON_HEDLEY_ASSUME(0)), (value)) + #else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE() + #endif +#else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (value) +#endif +#if !defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif + +JSON_HEDLEY_DIAGNOSTIC_PUSH +#if JSON_HEDLEY_HAS_WARNING("-Wpedantic") + #pragma clang diagnostic ignored "-Wpedantic" +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat-pedantic") && defined(__cplusplus) + #pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif +#if JSON_HEDLEY_GCC_HAS_WARNING("-Wvariadic-macros",4,0,0) + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wvariadic-macros" + #elif defined(JSON_HEDLEY_GCC_VERSION) + #pragma GCC diagnostic ignored "-Wvariadic-macros" + #endif +#endif +#if defined(JSON_HEDLEY_NON_NULL) + #undef JSON_HEDLEY_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) +#else + #define JSON_HEDLEY_NON_NULL(...) +#endif +JSON_HEDLEY_DIAGNOSTIC_POP + +#if defined(JSON_HEDLEY_PRINTF_FORMAT) + #undef JSON_HEDLEY_PRINTF_FORMAT +#endif +#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check))) +#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check))) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(format) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check))) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6,0,0) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check)) +#else + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) +#endif + +#if defined(JSON_HEDLEY_CONSTEXPR) + #undef JSON_HEDLEY_CONSTEXPR +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_CONSTEXPR JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(constexpr) + #endif +#endif +#if !defined(JSON_HEDLEY_CONSTEXPR) + #define JSON_HEDLEY_CONSTEXPR +#endif + +#if defined(JSON_HEDLEY_PREDICT) + #undef JSON_HEDLEY_PREDICT +#endif +#if defined(JSON_HEDLEY_LIKELY) + #undef JSON_HEDLEY_LIKELY +#endif +#if defined(JSON_HEDLEY_UNLIKELY) + #undef JSON_HEDLEY_UNLIKELY +#endif +#if defined(JSON_HEDLEY_UNPREDICTABLE) + #undef JSON_HEDLEY_UNPREDICTABLE +#endif +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable) + #define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable((expr)) +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) && !defined(JSON_HEDLEY_PGI_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(9,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability( (expr), (value), (probability)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1 , (probability)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0 , (probability)) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect (!!(expr), 1 ) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect (!!(expr), 0 ) +#elif \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,27) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, expected, probability) \ + (((probability) >= 0.9) ? __builtin_expect((expr), (expected)) : (JSON_HEDLEY_STATIC_CAST(void, expected), (expr))) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ + })) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ + })) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#else +# define JSON_HEDLEY_PREDICT(expr, expected, probability) (JSON_HEDLEY_STATIC_CAST(void, expected), (expr)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_LIKELY(expr) (!!(expr)) +# define JSON_HEDLEY_UNLIKELY(expr) (!!(expr)) +#endif +#if !defined(JSON_HEDLEY_UNPREDICTABLE) + #define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5) +#endif + +#if defined(JSON_HEDLEY_MALLOC) + #undef JSON_HEDLEY_MALLOC +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_MALLOC __attribute__((__malloc__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_MALLOC _Pragma("returns_new_memory") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_MALLOC __declspec(restrict) +#else + #define JSON_HEDLEY_MALLOC +#endif + +#if defined(JSON_HEDLEY_PURE) + #undef JSON_HEDLEY_PURE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(pure) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,96,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PURE __attribute__((__pure__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) +# define JSON_HEDLEY_PURE _Pragma("does_not_write_global_data") +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) \ + ) +# define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;") +#else +# define JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_CONST) + #undef JSON_HEDLEY_CONST +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(const) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_CONST __attribute__((__const__)) +#elif \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_CONST _Pragma("no_side_effect") +#else + #define JSON_HEDLEY_CONST JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_RESTRICT) + #undef JSON_HEDLEY_RESTRICT +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT restrict +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,4) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + defined(__clang__) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RESTRICT __restrict +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT _Restrict +#else + #define JSON_HEDLEY_RESTRICT +#endif + +#if defined(JSON_HEDLEY_INLINE) + #undef JSON_HEDLEY_INLINE +#endif +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + (defined(__cplusplus) && (__cplusplus >= 199711L)) + #define JSON_HEDLEY_INLINE inline +#elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(6,2,0) + #define JSON_HEDLEY_INLINE __inline__ +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,1,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_INLINE __inline +#else + #define JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_ALWAYS_INLINE) + #undef JSON_HEDLEY_ALWAYS_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) +# define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_ALWAYS_INLINE __forceinline +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) \ + ) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") +#else +# define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_NEVER_INLINE) + #undef JSON_HEDLEY_NEVER_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(10,2,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute((noinline)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#else + #define JSON_HEDLEY_NEVER_INLINE +#endif + +#if defined(JSON_HEDLEY_PRIVATE) + #undef JSON_HEDLEY_PRIVATE +#endif +#if defined(JSON_HEDLEY_PUBLIC) + #undef JSON_HEDLEY_PUBLIC +#endif +#if defined(JSON_HEDLEY_IMPORT) + #undef JSON_HEDLEY_IMPORT +#endif +#if defined(_WIN32) || defined(__CYGWIN__) +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC __declspec(dllexport) +# define JSON_HEDLEY_IMPORT __declspec(dllimport) +#else +# if \ + JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + ( \ + defined(__TI_EABI__) && \ + ( \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) \ + ) \ + ) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) +# define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default"))) +# else +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC +# endif +# define JSON_HEDLEY_IMPORT extern +#endif + +#if defined(JSON_HEDLEY_NO_THROW) + #undef JSON_HEDLEY_NO_THROW +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NO_THROW __declspec(nothrow) +#else + #define JSON_HEDLEY_NO_THROW +#endif + +#if defined(JSON_HEDLEY_FALL_THROUGH) + #undef JSON_HEDLEY_FALL_THROUGH +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(fallthrough) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(clang,fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[clang::fallthrough]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[fallthrough]]) +#elif defined(__fallthrough) /* SAL */ + #define JSON_HEDLEY_FALL_THROUGH __fallthrough +#else + #define JSON_HEDLEY_FALL_THROUGH +#endif + +#if defined(JSON_HEDLEY_RETURNS_NON_NULL) + #undef JSON_HEDLEY_RETURNS_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) +#elif defined(_Ret_notnull_) /* SAL */ + #define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_ +#else + #define JSON_HEDLEY_RETURNS_NON_NULL +#endif + +#if defined(JSON_HEDLEY_ARRAY_PARAM) + #undef JSON_HEDLEY_ARRAY_PARAM +#endif +#if \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + !defined(__STDC_NO_VLA__) && \ + !defined(__cplusplus) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_ARRAY_PARAM(name) (name) +#else + #define JSON_HEDLEY_ARRAY_PARAM(name) +#endif + +#if defined(JSON_HEDLEY_IS_CONSTANT) + #undef JSON_HEDLEY_IS_CONSTANT +#endif +#if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR) + #undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#endif +/* JSON_HEDLEY_IS_CONSTEXPR_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #undef JSON_HEDLEY_IS_CONSTEXPR_ +#endif +#if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) && !defined(__cplusplus)) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) +#endif +#if !defined(__cplusplus) +# if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,24) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*) +#endif +# elif \ + ( \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ + !defined(JSON_HEDLEY_SUNPRO_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION)) || \ + (JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,3,0) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0) +#endif +# elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + defined(JSON_HEDLEY_INTEL_VERSION) || \ + defined(JSON_HEDLEY_TINYC_VERSION) || \ + defined(JSON_HEDLEY_TI_ARMCL_VERSION) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(18,12,0) || \ + defined(JSON_HEDLEY_TI_CL2000_VERSION) || \ + defined(JSON_HEDLEY_TI_CL6X_VERSION) || \ + defined(JSON_HEDLEY_TI_CL7X_VERSION) || \ + defined(JSON_HEDLEY_TI_CLPRU_VERSION) || \ + defined(__clang__) +# define JSON_HEDLEY_IS_CONSTEXPR_(expr) ( \ + sizeof(void) != \ + sizeof(*( \ + 1 ? \ + ((void*) ((expr) * 0L) ) : \ +((struct { char v[sizeof(void) * 2]; } *) 1) \ + ) \ + ) \ + ) +# endif +#endif +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY_IS_CONSTEXPR_(expr) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY_IS_CONSTEXPR_(expr) ? (expr) : (-1)) +#else + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) (0) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) +#endif + +#if defined(JSON_HEDLEY_BEGIN_C_DECLS) + #undef JSON_HEDLEY_BEGIN_C_DECLS +#endif +#if defined(JSON_HEDLEY_END_C_DECLS) + #undef JSON_HEDLEY_END_C_DECLS +#endif +#if defined(JSON_HEDLEY_C_DECL) + #undef JSON_HEDLEY_C_DECL +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_BEGIN_C_DECLS extern "C" { + #define JSON_HEDLEY_END_C_DECLS } + #define JSON_HEDLEY_C_DECL extern "C" +#else + #define JSON_HEDLEY_BEGIN_C_DECLS + #define JSON_HEDLEY_END_C_DECLS + #define JSON_HEDLEY_C_DECL +#endif + +#if defined(JSON_HEDLEY_STATIC_ASSERT) + #undef JSON_HEDLEY_STATIC_ASSERT +#endif +#if \ + !defined(__cplusplus) && ( \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \ + (JSON_HEDLEY_HAS_FEATURE(c_static_assert) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(6,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + defined(_Static_assert) \ + ) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) +#elif \ + (defined(__cplusplus) && (__cplusplus >= 201103L)) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(static_assert(expr, message)) +#else +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) +#endif + +#if defined(JSON_HEDLEY_NULL) + #undef JSON_HEDLEY_NULL +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_NULL JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(nullptr) + #elif defined(NULL) + #define JSON_HEDLEY_NULL NULL + #else + #define JSON_HEDLEY_NULL JSON_HEDLEY_STATIC_CAST(void*, 0) + #endif +#elif defined(NULL) + #define JSON_HEDLEY_NULL NULL +#else + #define JSON_HEDLEY_NULL ((void*) 0) +#endif + +#if defined(JSON_HEDLEY_MESSAGE) + #undef JSON_HEDLEY_MESSAGE +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_MESSAGE(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(message msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg) +#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_WARNING) + #undef JSON_HEDLEY_WARNING +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_WARNING(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(clang warning msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,8,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_REQUIRE) + #undef JSON_HEDLEY_REQUIRE +#endif +#if defined(JSON_HEDLEY_REQUIRE_MSG) + #undef JSON_HEDLEY_REQUIRE_MSG +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if) +# if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat") +# define JSON_HEDLEY_REQUIRE(expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), #expr, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), msg, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_REQUIRE(expr) __attribute__((diagnose_if(!(expr), #expr, "error"))) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) __attribute__((diagnose_if(!(expr), msg, "error"))) +# endif +#else +# define JSON_HEDLEY_REQUIRE(expr) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) +#endif + +#if defined(JSON_HEDLEY_FLAGS) + #undef JSON_HEDLEY_FLAGS +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) && (!defined(__cplusplus) || JSON_HEDLEY_HAS_WARNING("-Wbitfield-enum-conversion")) + #define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__)) +#else + #define JSON_HEDLEY_FLAGS +#endif + +#if defined(JSON_HEDLEY_FLAGS_CAST) + #undef JSON_HEDLEY_FLAGS_CAST +#endif +#if JSON_HEDLEY_INTEL_VERSION_CHECK(19,0,0) +# define JSON_HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("warning(disable:188)") \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr) +#endif + +#if defined(JSON_HEDLEY_EMPTY_BASES) + #undef JSON_HEDLEY_EMPTY_BASES +#endif +#if \ + (JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,23918) && !JSON_HEDLEY_MSVC_VERSION_CHECK(20,0,0)) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_EMPTY_BASES __declspec(empty_bases) +#else + #define JSON_HEDLEY_EMPTY_BASES +#endif + +/* Remaining macros are deprecated. */ + +#if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#endif +#if defined(__clang__) + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0) +#else + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN) + #undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#endif +#define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin) + +#if defined(JSON_HEDLEY_CLANG_HAS_FEATURE) + #undef JSON_HEDLEY_CLANG_HAS_FEATURE +#endif +#define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature) + +#if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION) + #undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#endif +#define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension) + +#if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_WARNING) + #undef JSON_HEDLEY_CLANG_HAS_WARNING +#endif +#define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning) + +#endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */ + + +// This file contains all internal macro definitions (except those affecting ABI) +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// #include + + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// C++ language standard detection +// if the user manually specified the used c++ version this is skipped +#if !defined(JSON_HAS_CPP_23) && !defined(JSON_HAS_CPP_20) && !defined(JSON_HAS_CPP_17) && !defined(JSON_HAS_CPP_14) && !defined(JSON_HAS_CPP_11) + #if (defined(__cplusplus) && __cplusplus > 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG > 202002L) + #define JSON_HAS_CPP_23 + #define JSON_HAS_CPP_20 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus > 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG > 201703L) + #define JSON_HAS_CPP_20 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus > 201402L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus > 201103L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 + #endif + // the cpp 11 flag is always specified because it is the minimal required version + #define JSON_HAS_CPP_11 +#endif + +#ifdef __has_include + #if __has_include() + #include + #endif +#endif + +#if !defined(JSON_HAS_FILESYSTEM) && !defined(JSON_HAS_EXPERIMENTAL_FILESYSTEM) + #ifdef JSON_HAS_CPP_17 + #if defined(__cpp_lib_filesystem) + #define JSON_HAS_FILESYSTEM 1 + #elif defined(__cpp_lib_experimental_filesystem) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif !defined(__has_include) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #endif + + // std::filesystem does not work on MinGW GCC 8: https://sourceforge.net/p/mingw-w64/bugs/737/ + #if defined(__MINGW32__) && defined(__GNUC__) && __GNUC__ == 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before GCC 8: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before Clang 7: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__clang_major__) && __clang_major__ < 7 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before MSVC 19.14: https://en.cppreference.com/w/cpp/compiler_support + #if defined(_MSC_VER) && _MSC_VER < 1914 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before iOS 13 + #if defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED < 130000 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before macOS Catalina + #if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + #endif +#endif + +#ifndef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_FILESYSTEM + #define JSON_HAS_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_THREE_WAY_COMPARISON + #if defined(__cpp_impl_three_way_comparison) && __cpp_impl_three_way_comparison >= 201907L \ + && defined(__cpp_lib_three_way_comparison) && __cpp_lib_three_way_comparison >= 201907L + #define JSON_HAS_THREE_WAY_COMPARISON 1 + #else + #define JSON_HAS_THREE_WAY_COMPARISON 0 + #endif +#endif + +#ifndef JSON_HAS_RANGES + // ranges header shipping in GCC 11.1.0 (released 2021-04-27) has syntax error + #if defined(__GLIBCXX__) && __GLIBCXX__ == 20210427 + #define JSON_HAS_RANGES 0 + #elif defined(__cpp_lib_ranges) + #define JSON_HAS_RANGES 1 + #else + #define JSON_HAS_RANGES 0 + #endif +#endif + +#ifndef JSON_HAS_STATIC_RTTI + #if !defined(_HAS_STATIC_RTTI) || _HAS_STATIC_RTTI != 0 + #define JSON_HAS_STATIC_RTTI 1 + #else + #define JSON_HAS_STATIC_RTTI 0 + #endif +#endif + +#ifdef JSON_HAS_CPP_17 + #define JSON_INLINE_VARIABLE inline +#else + #define JSON_INLINE_VARIABLE +#endif + +#if JSON_HEDLEY_HAS_ATTRIBUTE(no_unique_address) + #define JSON_NO_UNIQUE_ADDRESS [[no_unique_address]] +#else + #define JSON_NO_UNIQUE_ADDRESS +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdocumentation" + #pragma clang diagnostic ignored "-Wdocumentation-unknown-command" +#endif + +// allow disabling exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #include + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// allow overriding assert +#if !defined(JSON_ASSERT) + #include // assert + #define JSON_ASSERT(x) assert(x) +#endif + +// allow to access some private functions (needed by the test suite) +#if defined(JSON_TESTS_PRIVATE) + #define JSON_PRIVATE_UNLESS_TESTED public +#else + #define JSON_PRIVATE_UNLESS_TESTED private +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + /* NOLINTNEXTLINE(modernize-type-traits) we use C++11 */ \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + /* NOLINTNEXTLINE(modernize-avoid-c-arrays) we don't want to depend on */ \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + /* NOLINTNEXTLINE(modernize-type-traits) we use C++11 */ \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + /* NOLINTNEXTLINE(modernize-avoid-c-arrays) we don't want to depend on */ \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [&j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer, \ + class BinaryType, \ + class CustomBaseClass> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// Macros to simplify conversion from/to types + +#define NLOHMANN_JSON_EXPAND( x ) x +#define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME +#define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ + NLOHMANN_JSON_PASTE64, \ + NLOHMANN_JSON_PASTE63, \ + NLOHMANN_JSON_PASTE62, \ + NLOHMANN_JSON_PASTE61, \ + NLOHMANN_JSON_PASTE60, \ + NLOHMANN_JSON_PASTE59, \ + NLOHMANN_JSON_PASTE58, \ + NLOHMANN_JSON_PASTE57, \ + NLOHMANN_JSON_PASTE56, \ + NLOHMANN_JSON_PASTE55, \ + NLOHMANN_JSON_PASTE54, \ + NLOHMANN_JSON_PASTE53, \ + NLOHMANN_JSON_PASTE52, \ + NLOHMANN_JSON_PASTE51, \ + NLOHMANN_JSON_PASTE50, \ + NLOHMANN_JSON_PASTE49, \ + NLOHMANN_JSON_PASTE48, \ + NLOHMANN_JSON_PASTE47, \ + NLOHMANN_JSON_PASTE46, \ + NLOHMANN_JSON_PASTE45, \ + NLOHMANN_JSON_PASTE44, \ + NLOHMANN_JSON_PASTE43, \ + NLOHMANN_JSON_PASTE42, \ + NLOHMANN_JSON_PASTE41, \ + NLOHMANN_JSON_PASTE40, \ + NLOHMANN_JSON_PASTE39, \ + NLOHMANN_JSON_PASTE38, \ + NLOHMANN_JSON_PASTE37, \ + NLOHMANN_JSON_PASTE36, \ + NLOHMANN_JSON_PASTE35, \ + NLOHMANN_JSON_PASTE34, \ + NLOHMANN_JSON_PASTE33, \ + NLOHMANN_JSON_PASTE32, \ + NLOHMANN_JSON_PASTE31, \ + NLOHMANN_JSON_PASTE30, \ + NLOHMANN_JSON_PASTE29, \ + NLOHMANN_JSON_PASTE28, \ + NLOHMANN_JSON_PASTE27, \ + NLOHMANN_JSON_PASTE26, \ + NLOHMANN_JSON_PASTE25, \ + NLOHMANN_JSON_PASTE24, \ + NLOHMANN_JSON_PASTE23, \ + NLOHMANN_JSON_PASTE22, \ + NLOHMANN_JSON_PASTE21, \ + NLOHMANN_JSON_PASTE20, \ + NLOHMANN_JSON_PASTE19, \ + NLOHMANN_JSON_PASTE18, \ + NLOHMANN_JSON_PASTE17, \ + NLOHMANN_JSON_PASTE16, \ + NLOHMANN_JSON_PASTE15, \ + NLOHMANN_JSON_PASTE14, \ + NLOHMANN_JSON_PASTE13, \ + NLOHMANN_JSON_PASTE12, \ + NLOHMANN_JSON_PASTE11, \ + NLOHMANN_JSON_PASTE10, \ + NLOHMANN_JSON_PASTE9, \ + NLOHMANN_JSON_PASTE8, \ + NLOHMANN_JSON_PASTE7, \ + NLOHMANN_JSON_PASTE6, \ + NLOHMANN_JSON_PASTE5, \ + NLOHMANN_JSON_PASTE4, \ + NLOHMANN_JSON_PASTE3, \ + NLOHMANN_JSON_PASTE2, \ + NLOHMANN_JSON_PASTE1)(__VA_ARGS__)) +#define NLOHMANN_JSON_PASTE2(func, v1) func(v1) +#define NLOHMANN_JSON_PASTE3(func, v1, v2) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE2(func, v2) +#define NLOHMANN_JSON_PASTE4(func, v1, v2, v3) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE3(func, v2, v3) +#define NLOHMANN_JSON_PASTE5(func, v1, v2, v3, v4) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE4(func, v2, v3, v4) +#define NLOHMANN_JSON_PASTE6(func, v1, v2, v3, v4, v5) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE5(func, v2, v3, v4, v5) +#define NLOHMANN_JSON_PASTE7(func, v1, v2, v3, v4, v5, v6) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE6(func, v2, v3, v4, v5, v6) +#define NLOHMANN_JSON_PASTE8(func, v1, v2, v3, v4, v5, v6, v7) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE7(func, v2, v3, v4, v5, v6, v7) +#define NLOHMANN_JSON_PASTE9(func, v1, v2, v3, v4, v5, v6, v7, v8) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE8(func, v2, v3, v4, v5, v6, v7, v8) +#define NLOHMANN_JSON_PASTE10(func, v1, v2, v3, v4, v5, v6, v7, v8, v9) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE9(func, v2, v3, v4, v5, v6, v7, v8, v9) +#define NLOHMANN_JSON_PASTE11(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE10(func, v2, v3, v4, v5, v6, v7, v8, v9, v10) +#define NLOHMANN_JSON_PASTE12(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE11(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) +#define NLOHMANN_JSON_PASTE13(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE12(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) +#define NLOHMANN_JSON_PASTE14(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE13(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) +#define NLOHMANN_JSON_PASTE15(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE14(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) +#define NLOHMANN_JSON_PASTE16(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE15(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) +#define NLOHMANN_JSON_PASTE17(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE16(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) +#define NLOHMANN_JSON_PASTE18(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE17(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) +#define NLOHMANN_JSON_PASTE19(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE18(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) +#define NLOHMANN_JSON_PASTE20(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE19(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) +#define NLOHMANN_JSON_PASTE21(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE20(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) +#define NLOHMANN_JSON_PASTE22(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE21(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) +#define NLOHMANN_JSON_PASTE23(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE22(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) +#define NLOHMANN_JSON_PASTE24(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE23(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) +#define NLOHMANN_JSON_PASTE25(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE24(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) +#define NLOHMANN_JSON_PASTE26(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE25(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) +#define NLOHMANN_JSON_PASTE27(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE26(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) +#define NLOHMANN_JSON_PASTE28(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE27(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) +#define NLOHMANN_JSON_PASTE29(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE28(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) +#define NLOHMANN_JSON_PASTE30(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE29(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) +#define NLOHMANN_JSON_PASTE31(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE30(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) +#define NLOHMANN_JSON_PASTE32(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE31(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) +#define NLOHMANN_JSON_PASTE33(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE32(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) +#define NLOHMANN_JSON_PASTE34(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE33(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) +#define NLOHMANN_JSON_PASTE35(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE34(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) +#define NLOHMANN_JSON_PASTE36(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE35(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) +#define NLOHMANN_JSON_PASTE37(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE36(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) +#define NLOHMANN_JSON_PASTE38(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE37(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) +#define NLOHMANN_JSON_PASTE39(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE38(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) +#define NLOHMANN_JSON_PASTE40(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE39(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) +#define NLOHMANN_JSON_PASTE41(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE40(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) +#define NLOHMANN_JSON_PASTE42(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE41(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) +#define NLOHMANN_JSON_PASTE43(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE42(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) +#define NLOHMANN_JSON_PASTE44(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE43(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) +#define NLOHMANN_JSON_PASTE45(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE44(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) +#define NLOHMANN_JSON_PASTE46(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE45(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) +#define NLOHMANN_JSON_PASTE47(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE46(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) +#define NLOHMANN_JSON_PASTE48(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE47(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) +#define NLOHMANN_JSON_PASTE49(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE48(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) +#define NLOHMANN_JSON_PASTE50(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE49(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) +#define NLOHMANN_JSON_PASTE51(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE50(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) +#define NLOHMANN_JSON_PASTE52(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE51(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) +#define NLOHMANN_JSON_PASTE53(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE52(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) +#define NLOHMANN_JSON_PASTE54(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE53(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) +#define NLOHMANN_JSON_PASTE55(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE54(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) +#define NLOHMANN_JSON_PASTE56(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE55(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) +#define NLOHMANN_JSON_PASTE57(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE56(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) +#define NLOHMANN_JSON_PASTE58(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE57(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) +#define NLOHMANN_JSON_PASTE59(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE58(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) +#define NLOHMANN_JSON_PASTE60(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE59(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) +#define NLOHMANN_JSON_PASTE61(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE60(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) +#define NLOHMANN_JSON_PASTE62(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE61(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) +#define NLOHMANN_JSON_PASTE63(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE62(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) +#define NLOHMANN_JSON_PASTE64(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE63(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) + +#define NLOHMANN_JSON_TO(v1) nlohmann_json_j[#v1] = nlohmann_json_t.v1; +#define NLOHMANN_JSON_FROM(v1) nlohmann_json_j.at(#v1).get_to(nlohmann_json_t.v1); +#define NLOHMANN_JSON_FROM_WITH_DEFAULT(v1) nlohmann_json_t.v1 = !nlohmann_json_j.is_null() ? nlohmann_json_j.value(#v1, nlohmann_json_default_obj.v1) : nlohmann_json_default_obj.v1; + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE +@since version 3.9.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE(Type, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + friend void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT +@since version 3.11.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + friend void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE +@since version 3.11.3 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE +@since version 3.9.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_non_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Type, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT +@since version 3.11.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_non_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE +@since version 3.11.3 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_type_non_intrusive/ +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE(Type, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE(Type, BaseType, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + friend void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { nlohmann::from_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_WITH_DEFAULT +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_WITH_DEFAULT(Type, BaseType, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + friend void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { nlohmann::from_json(nlohmann_json_j, static_cast(nlohmann_json_t)); const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_ONLY_SERIALIZE +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_ONLY_SERIALIZE(Type, BaseType, ...) \ + template::value, int> = 0> \ + friend void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE(Type, BaseType, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { nlohmann::from_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_WITH_DEFAULT +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, BaseType, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + template::value, int> = 0> \ + void from_json(const BasicJsonType& nlohmann_json_j, Type& nlohmann_json_t) { nlohmann::from_json(nlohmann_json_j, static_cast(nlohmann_json_t)); const Type nlohmann_json_default_obj{}; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE +@since version 3.12.0 +@sa https://json.nlohmann.me/api/macros/nlohmann_define_derived_type/ +*/ +#define NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE(Type, BaseType, ...) \ + template::value, int> = 0> \ + void to_json(BasicJsonType& nlohmann_json_j, const Type& nlohmann_json_t) { nlohmann::to_json(nlohmann_json_j, static_cast(nlohmann_json_t)); NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } + +// inspired from https://stackoverflow.com/a/26745591 +// allows calling any std function as if (e.g., with begin): +// using std::begin; begin(x); +// +// it allows using the detected idiom to retrieve the return type +// of such an expression +#define NLOHMANN_CAN_CALL_STD_FUNC_IMPL(std_name) \ + namespace detail { \ + using std::std_name; \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + } \ + \ + namespace detail2 { \ + struct std_name##_tag \ + { \ + }; \ + \ + template \ + std_name##_tag std_name(T&&...); \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + \ + template \ + struct would_call_std_##std_name \ + { \ + static constexpr auto const value = ::nlohmann::detail:: \ + is_detected_exact::value; \ + }; \ + } /* namespace detail2 */ \ + \ + template \ + struct would_call_std_##std_name : detail2::would_call_std_##std_name \ + { \ + } + +#ifndef JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_USE_IMPLICIT_CONVERSIONS 1 +#endif + +#if JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_EXPLICIT +#else + #define JSON_EXPLICIT explicit +#endif + +#ifndef JSON_DISABLE_ENUM_SERIALIZATION + #define JSON_DISABLE_ENUM_SERIALIZATION 0 +#endif + +#ifndef JSON_USE_GLOBAL_UDLS + #define JSON_USE_GLOBAL_UDLS 1 +#endif + +#if JSON_HAS_THREE_WAY_COMPARISON + #include // partial_ordering +#endif + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/////////////////////////// +// JSON type enumeration // +/////////////////////////// + +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa see @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type + +@since version 1.0.0 +*/ +enum class value_t : std::uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + binary, ///< binary array (ordered collection of bytes) + discarded ///< discarded by the parser callback function +}; + +/*! +@brief comparison operator for JSON types + +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string < binary +- furthermore, each type is not smaller than itself +- discarded values are not comparable +- binary is represented as a b"" string in python and directly comparable to a + string; however, making a binary array directly comparable with a string would + be surprising behavior in a JSON file. + +@since version 1.0.0 +*/ +#if JSON_HAS_THREE_WAY_COMPARISON + inline std::partial_ordering operator<=>(const value_t lhs, const value_t rhs) noexcept // *NOPAD* +#else + inline bool operator<(const value_t lhs, const value_t rhs) noexcept +#endif +{ + static constexpr std::array order = {{ + 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, + 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */, + 6 /* binary */ + } + }; + + const auto l_index = static_cast(lhs); + const auto r_index = static_cast(rhs); +#if JSON_HAS_THREE_WAY_COMPARISON + if (l_index < order.size() && r_index < order.size()) + { + return order[l_index] <=> order[r_index]; // *NOPAD* + } + return std::partial_ordering::unordered; +#else + return l_index < order.size() && r_index < order.size() && order[l_index] < order[r_index]; +#endif +} + +// GCC selects the built-in operator< over an operator rewritten from +// a user-defined spaceship operator +// Clang, MSVC, and ICC select the rewritten candidate +// (see GCC bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105200) +#if JSON_HAS_THREE_WAY_COMPARISON && defined(__GNUC__) +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + return std::is_lt(lhs <=> rhs); // *NOPAD* +} +#endif + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief replace all occurrences of a substring by another string + +@param[in,out] s the string to manipulate; changed so that all + occurrences of @a f are replaced with @a t +@param[in] f the substring to replace with @a t +@param[in] t the string to replace @a f + +@pre The search string @a f must not be empty. **This precondition is +enforced with an assertion.** + +@since version 2.0.0 +*/ +template +inline void replace_substring(StringType& s, const StringType& f, + const StringType& t) +{ + JSON_ASSERT(!f.empty()); + for (auto pos = s.find(f); // find first occurrence of f + pos != StringType::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t, and + pos = s.find(f, pos + t.size())) // find next occurrence of f + {} +} + +/*! + * @brief string escaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to escape + * @return escaped string + * + * Note the order of escaping "~" to "~0" and "/" to "~1" is important. + */ +template +inline StringType escape(StringType s) +{ + replace_substring(s, StringType{"~"}, StringType{"~0"}); + replace_substring(s, StringType{"/"}, StringType{"~1"}); + return s; +} + +/*! + * @brief string unescaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to unescape + * @return unescaped string + * + * Note the order of escaping "~1" to "/" and "~0" to "~" is important. + */ +template +static void unescape(StringType& s) +{ + replace_substring(s, StringType{"~1"}, StringType{"/"}); + replace_substring(s, StringType{"~0"}, StringType{"~"}); +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-FileCopyrightText: 2018 The Abseil Authors +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type +#include // index_sequence, make_index_sequence, index_sequence_for + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template +using uncvref_t = typename std::remove_cv::type>::type; + +#ifdef JSON_HAS_CPP_14 + +// the following utilities are natively available in C++14 +using std::enable_if_t; +using std::index_sequence; +using std::make_index_sequence; +using std::index_sequence_for; + +#else + +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +// The following code is taken from https://github.com/abseil/abseil-cpp/blob/10cb35e459f5ecca5b2ff107635da0bfa41011b4/absl/utility/utility.h +// which is part of Google Abseil (https://github.com/abseil/abseil-cpp), licensed under the Apache License 2.0. + +//// START OF CODE FROM GOOGLE ABSEIL + +// integer_sequence +// +// Class template representing a compile-time integer sequence. An instantiation +// of `integer_sequence` has a sequence of integers encoded in its +// type through its template arguments (which is a common need when +// working with C++11 variadic templates). `absl::integer_sequence` is designed +// to be a drop-in replacement for C++14's `std::integer_sequence`. +// +// Example: +// +// template< class T, T... Ints > +// void user_function(integer_sequence); +// +// int main() +// { +// // user_function's `T` will be deduced to `int` and `Ints...` +// // will be deduced to `0, 1, 2, 3, 4`. +// user_function(make_integer_sequence()); +// } +template +struct integer_sequence +{ + using value_type = T; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +// index_sequence +// +// A helper template for an `integer_sequence` of `size_t`, +// `absl::index_sequence` is designed to be a drop-in replacement for C++14's +// `std::index_sequence`. +template +using index_sequence = integer_sequence; + +namespace utility_internal +{ + +template +struct Extend; + +// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. +template +struct Extend, SeqSize, 0> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)... >; +}; + +template +struct Extend, SeqSize, 1> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)..., 2 * SeqSize >; +}; + +// Recursion helper for 'make_integer_sequence'. +// 'Gen::type' is an alias for 'integer_sequence'. +template +struct Gen +{ + using type = + typename Extend < typename Gen < T, N / 2 >::type, N / 2, N % 2 >::type; +}; + +template +struct Gen +{ + using type = integer_sequence; +}; + +} // namespace utility_internal + +// Compile-time sequences of integers + +// make_integer_sequence +// +// This template alias is equivalent to +// `integer_sequence`, and is designed to be a drop-in +// replacement for C++14's `std::make_integer_sequence`. +template +using make_integer_sequence = typename utility_internal::Gen::type; + +// make_index_sequence +// +// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, +// and is designed to be a drop-in replacement for C++14's +// `std::make_index_sequence`. +template +using make_index_sequence = make_integer_sequence; + +// index_sequence_for +// +// Converts a typename pack into an index sequence of the same length, and +// is designed to be a drop-in replacement for C++14's +// `std::index_sequence_for()` +template +using index_sequence_for = make_index_sequence; + +//// END OF CODE FROM GOOGLE ABSEIL + +#endif + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static JSON_INLINE_VARIABLE constexpr T value{}; +}; + +#ifndef JSON_HAS_CPP_17 + template + constexpr T static_const::value; +#endif + +template +constexpr std::array make_array(Args&& ... args) +{ + return std::array {{static_cast(std::forward(args))...}}; +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // numeric_limits +#include // char_traits +#include // tuple +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // random_access_iterator_tag + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN + +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin); + +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN + +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end); + +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann +// SPDX-License-Identifier: MIT + +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ + #define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + #include // int64_t, uint64_t + #include // map + #include // allocator + #include // string + #include // vector + + // #include + + + /*! + @brief namespace for Niels Lohmann + @see https://github.com/nlohmann + @since version 1.0.0 + */ + NLOHMANN_JSON_NAMESPACE_BEGIN + + /*! + @brief default JSONSerializer template argument + + This serializer ignores the template arguments and uses ADL + ([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) + for serialization. + */ + template + struct adl_serializer; + + /// a class to store JSON values + /// @sa https://json.nlohmann.me/api/basic_json/ + template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer, + class BinaryType = std::vector, // cppcheck-suppress syntaxError + class CustomBaseClass = void> + class basic_json; + + /// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document + /// @sa https://json.nlohmann.me/api/json_pointer/ + template + class json_pointer; + + /*! + @brief default specialization + @sa https://json.nlohmann.me/api/json/ + */ + using json = basic_json<>; + + /// @brief a minimal map-like container that preserves insertion order + /// @sa https://json.nlohmann.me/api/ordered_map/ + template + struct ordered_map; + + /// @brief specialization that maintains the insertion order of object keys + /// @sa https://json.nlohmann.me/api/ordered_json/ + using ordered_json = basic_json; + + NLOHMANN_JSON_NAMESPACE_END + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + +NLOHMANN_JSON_NAMESPACE_BEGIN +/*! +@brief detail namespace with internal helper functions + +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. + +@since version 2.1.0 +*/ +namespace detail +{ + +///////////// +// helpers // +///////////// + +// Note to maintainers: +// +// Every trait in this file expects a non CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e. those of the form: decltype(T::member_function(std::declval()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g. to_json(BasicJsonType&, const T&)) + +template struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json : std::true_type {}; + +// used by exceptions create() member functions +// true_type for pointer to possibly cv-qualified basic_json or std::nullptr_t +// false_type otherwise +template +struct is_basic_json_context : + std::integral_constant < bool, + is_basic_json::type>::type>::value + || std::is_same::value > +{}; + +////////////////////// +// json_ref helpers // +////////////////////// + +template +class json_ref; + +template +struct is_json_ref : std::false_type {}; + +template +struct is_json_ref> : std::true_type {}; + +////////////////////////// +// aliases for detected // +////////////////////////// + +template +using mapped_type_t = typename T::mapped_type; + +template +using key_type_t = typename T::key_type; + +template +using value_type_t = typename T::value_type; + +template +using difference_type_t = typename T::difference_type; + +template +using pointer_t = typename T::pointer; + +template +using reference_t = typename T::reference; + +template +using iterator_category_t = typename T::iterator_category; + +template +using to_json_function = decltype(T::to_json(std::declval()...)); + +template +using from_json_function = decltype(T::from_json(std::declval()...)); + +template +using get_template_function = decltype(std::declval().template get()); + +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json : std::false_type {}; + +// trait checking if j.get is valid +// use this trait instead of std::is_constructible or std::is_convertible, +// both rely on, or make use of implicit conversions, and thus fail when T +// has several constructors/operator= (see https://github.com/nlohmann/json/issues/958) +template +struct is_getable +{ + static constexpr bool value = is_detected::value; +}; + +template +struct has_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json : std::false_type {}; + +template +struct has_non_default_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if BasicJsonType::json_serializer::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template +struct has_to_json : std::false_type {}; + +template +struct has_to_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +template +using detect_key_compare = typename T::key_compare; + +template +struct has_key_compare : std::integral_constant::value> {}; + +// obtains the actual object key comparator +template +struct actual_object_comparator +{ + using object_t = typename BasicJsonType::object_t; + using object_comparator_t = typename BasicJsonType::default_object_comparator_t; + using type = typename std::conditional < has_key_compare::value, + typename object_t::key_compare, object_comparator_t>::type; +}; + +template +using actual_object_comparator_t = typename actual_object_comparator::type; + +///////////////// +// char_traits // +///////////////// + +// Primary template of char_traits calls std char_traits +template +struct char_traits : std::char_traits +{}; + +// Explicitly define char traits for unsigned char since it is not standard +template<> +struct char_traits : std::char_traits +{ + using char_type = unsigned char; + using int_type = uint64_t; + + // Redefine to_int_type function + static int_type to_int_type(char_type c) noexcept + { + return static_cast(c); + } + + static char_type to_char_type(int_type i) noexcept + { + return static_cast(i); + } + + static constexpr int_type eof() noexcept + { + return static_cast(std::char_traits::eof()); + } +}; + +// Explicitly define char traits for signed char since it is not standard +template<> +struct char_traits : std::char_traits +{ + using char_type = signed char; + using int_type = uint64_t; + + // Redefine to_int_type function + static int_type to_int_type(char_type c) noexcept + { + return static_cast(c); + } + + static char_type to_char_type(int_type i) noexcept + { + return static_cast(i); + } + + static constexpr int_type eof() noexcept + { + return static_cast(std::char_traits::eof()); + } +}; + +/////////////////// +// is_ functions // +/////////////////// + +// https://en.cppreference.com/w/cpp/types/conjunction +template struct conjunction : std::true_type { }; +template struct conjunction : B { }; +template +struct conjunction +: std::conditional(B::value), conjunction, B>::type {}; + +// https://en.cppreference.com/w/cpp/types/negation +template struct negation : std::integral_constant < bool, !B::value > { }; + +// Reimplementation of is_constructible and is_default_constructible, due to them being broken for +// std::pair and std::tuple until LWG 2367 fix (see https://cplusplus.github.io/LWG/lwg-defects.html#2367). +// This causes compile errors in e.g. clang 3.5 or gcc 4.9. +template +struct is_default_constructible : std::is_default_constructible {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + +template +struct is_constructible : std::is_constructible {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_iterator_traits : std::false_type {}; + +template +struct is_iterator_traits> +{ + private: + using traits = iterator_traits; + + public: + static constexpr auto value = + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value; +}; + +template +struct is_range +{ + private: + using t_ref = typename std::add_lvalue_reference::type; + + using iterator = detected_t; + using sentinel = detected_t; + + // to be 100% correct, it should use https://en.cppreference.com/w/cpp/iterator/input_or_output_iterator + // and https://en.cppreference.com/w/cpp/iterator/sentinel_for + // but reimplementing these would be too much work, as a lot of other concepts are used underneath + static constexpr auto is_iterator_begin = + is_iterator_traits>::value; + + public: + static constexpr bool value = !std::is_same::value && !std::is_same::value && is_iterator_begin; +}; + +template +using iterator_t = enable_if_t::value, result_of_begin())>>; + +template +using range_value_t = value_type_t>>; + +// The following implementation of is_complete_type is taken from +// https://blogs.msdn.microsoft.com/vcblog/2015/12/02/partial-support-for-expression-sfinae-in-vs-2015-update-1/ +// and is written by Xiang Fan who agreed to using it in this library. + +template +struct is_complete_type : std::false_type {}; + +template +struct is_complete_type : std::true_type {}; + +template +struct is_compatible_object_type_impl : std::false_type {}; + +template +struct is_compatible_object_type_impl < + BasicJsonType, CompatibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + // macOS's is_constructible does not play well with nonesuch... + static constexpr bool value = + is_constructible::value && + is_constructible::value; +}; + +template +struct is_compatible_object_type + : is_compatible_object_type_impl {}; + +template +struct is_constructible_object_type_impl : std::false_type {}; + +template +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (is_default_constructible::value && + (std::is_move_assignable::value || + std::is_copy_assignable::value) && + (is_constructible::value && + std::is_same < + typename object_t::mapped_type, + typename ConstructibleObjectType::mapped_type >::value)) || + (has_from_json::value || + has_non_default_from_json < + BasicJsonType, + typename ConstructibleObjectType::mapped_type >::value); +}; + +template +struct is_constructible_object_type + : is_constructible_object_type_impl {}; + +template +struct is_compatible_string_type +{ + static constexpr auto value = + is_constructible::value; +}; + +template +struct is_constructible_string_type +{ + // launder type through decltype() to fix compilation failure on ICPC +#ifdef __INTEL_COMPILER + using laundered_type = decltype(std::declval()); +#else + using laundered_type = ConstructibleStringType; +#endif + + static constexpr auto value = + conjunction < + is_constructible, + is_detected_exact>::value; +}; + +template +struct is_compatible_array_type_impl : std::false_type {}; + +template +struct is_compatible_array_type_impl < + BasicJsonType, CompatibleArrayType, + enable_if_t < + is_detected::value&& + is_iterator_traits>>::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 + !std::is_same>::value >> +{ + static constexpr bool value = + is_constructible>::value; +}; + +template +struct is_compatible_array_type + : is_compatible_array_type_impl {}; + +template +struct is_constructible_array_type_impl : std::false_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t::value >> + : std::true_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t < !std::is_same::value&& + !is_compatible_string_type::value&& + is_default_constructible::value&& +(std::is_move_assignable::value || + std::is_copy_assignable::value)&& +is_detected::value&& +is_iterator_traits>>::value&& +is_detected::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 +!std::is_same>::value&& +is_complete_type < +detected_t>::value >> +{ + using value_type = range_value_t; + + static constexpr bool value = + std::is_same::value || + has_from_json::value || + has_non_default_from_json < + BasicJsonType, + value_type >::value; +}; + +template +struct is_constructible_array_type + : is_constructible_array_type_impl {}; + +template +struct is_compatible_integer_type_impl : std::false_type {}; + +template +struct is_compatible_integer_type_impl < + RealIntegerType, CompatibleNumberIntegerType, + enable_if_t < std::is_integral::value&& + std::is_integral::value&& + !std::is_same::value >> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + is_constructible::value && + CompatibleLimits::is_integer && + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template +struct is_compatible_integer_type + : is_compatible_integer_type_impl {}; + +template +struct is_compatible_type_impl: std::false_type {}; + +template +struct is_compatible_type_impl < + BasicJsonType, CompatibleType, + enable_if_t::value >> +{ + static constexpr bool value = + has_to_json::value; +}; + +template +struct is_compatible_type + : is_compatible_type_impl {}; + +template +struct is_constructible_tuple : std::false_type {}; + +template +struct is_constructible_tuple> : conjunction...> {}; + +template +struct is_json_iterator_of : std::false_type {}; + +template +struct is_json_iterator_of : std::true_type {}; + +template +struct is_json_iterator_of : std::true_type +{}; + +// checks if a given type T is a template specialization of Primary +template